diff options
Diffstat (limited to 'src/common')
85 files changed, 6723 insertions, 2001 deletions
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index b8c5dd4fea..a1c819fffa 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \ - crypto_curve25519.obj curve25519-donna.obj +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \ + tortls.obj crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj diff --git a/src/common/address.c b/src/common/address.c index 96b99fa082..c7aa97bfd1 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -33,7 +33,7 @@ #include <process.h> #include <windows.h> #include <iphlpapi.h> -#endif +#endif /* defined(_WIN32) */ #include "compat.h" #include "util.h" @@ -159,6 +159,8 @@ tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa, tor_assert(a); tor_assert(sa); + /* This memset is redundant; leaving it in to avoid any future accidents, + however. */ memset(a, 0, sizeof(*a)); if (sa->sa_family == AF_INET) { @@ -196,7 +198,7 @@ tor_sockaddr_to_str(const struct sockaddr *sa) tor_asprintf(&result, "unix:%s", s_un->sun_path); return result; } -#endif +#endif /* defined(HAVE_SYS_UN_H) */ if (sa->sa_family == AF_UNSPEC) return tor_strdup("unspec"); @@ -235,8 +237,8 @@ tor_addr_make_null(tor_addr_t *a, sa_family_t family) * * Return 0 on success, -1 on failure; 1 on transient failure. */ -int -tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) +MOCK_IMPL(int, +tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) { /* Perhaps eventually this should be replaced by a tor_getaddrinfo or * something. @@ -303,7 +305,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) return result; } return (err == EAI_AGAIN) ? 1 : -1; -#else +#else /* !(defined(HAVE_GETADDRINFO)) */ struct hostent *ent; int err; #ifdef HAVE_GETHOSTBYNAME_R_6_ARG @@ -328,7 +330,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) #else err = h_errno; #endif -#endif /* endif HAVE_GETHOSTBYNAME_R_6_ARG. */ +#endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */ if (ent) { if (ent->h_addrtype == AF_INET) { tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); @@ -344,7 +346,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) #else return (err == TRY_AGAIN) ? 1 : -1; #endif -#endif +#endif /* defined(HAVE_GETADDRINFO) */ } } @@ -562,8 +564,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name, * and store the result in the <b>outlen</b>-byte buffer at - * <b>out</b>. Return the number of chars written to <b>out</b>, not - * including the trailing \0, on success. Returns -1 on failure. */ + * <b>out</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_addr_to_PTR_name(char *out, size_t outlen, const tor_addr_t *addr) @@ -905,8 +907,8 @@ tor_addr_is_loopback(const tor_addr_t *addr) return (tor_addr_to_ipv4h(addr) & 0xff000000) == 0x7f000000; case AF_UNSPEC: return 0; - default: /* LCOV_EXCL_START */ + default: tor_fragile_assert(); return 0; /* LCOV_EXCL_STOP */ @@ -1029,8 +1031,10 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src) memcpy(dest->addr.in6_addr.s6_addr, src->addr.in6_addr.s6_addr, 16); case AF_UNSPEC: break; + // LCOV_EXCL_START default: - tor_fragile_assert(); // LCOV_EXCL_LINE + tor_fragile_assert(); + // LCOV_EXCL_STOP } } @@ -1136,8 +1140,8 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, return 0; else return 1; - default: /* LCOV_EXCL_START */ + default: tor_fragile_assert(); return 0; /* LCOV_EXCL_STOP */ @@ -1195,8 +1199,8 @@ tor_addr_hash(const tor_addr_t *addr) return siphash24g(unspec_hash_input, sizeof(unspec_hash_input)); case AF_INET6: return siphash24g(&addr->addr.in6_addr.s6_addr, 16); - default: /* LCOV_EXCL_START */ + default: tor_fragile_assert(); return 0; /* LCOV_EXCL_STOP */ @@ -1432,7 +1436,7 @@ get_interface_addresses_ifaddrs(int severity, sa_family_t family) return result; } -#endif +#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */ #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST @@ -1523,7 +1527,7 @@ get_interface_addresses_win32(int severity, sa_family_t family) return result; } -#endif +#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */ #ifdef HAVE_IFCONF_TO_SMARTLIST @@ -1625,7 +1629,7 @@ get_interface_addresses_ioctl(int severity, sa_family_t family) tor_free(ifc.ifc_buf); return result; } -#endif +#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ /** Try to ask our network interfaces what addresses they are bound to. * Return a new smartlist of tor_addr_t on success, and NULL on failure. @@ -1804,9 +1808,10 @@ free_interface_address6_list(smartlist_t *addrs) * Returns NULL on failure. * Use free_interface_address6_list to free the returned list. */ -MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, - sa_family_t family, - int include_internal)) +MOCK_IMPL(smartlist_t *, +get_interface_address6_list,(int severity, + sa_family_t family, + int include_internal)) { smartlist_t *addrs; tor_addr_t addr; @@ -2074,7 +2079,8 @@ parse_port_range(const char *port, uint16_t *port_min_out, /** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual), * write it as a string into the <b>buf_len</b>-byte buffer in - * <b>buf</b>. + * <b>buf</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len) @@ -2125,7 +2131,8 @@ get_interface_address,(int severity, uint32_t *addr)) } /** Return true if we can tell that <b>name</b> is a canonical name for the - * loopback address. */ + * loopback address. Return true also for *.local hostnames, which are + * multicast DNS names for hosts on the local network. */ int tor_addr_hostname_is_local(const char *name) { @@ -2146,3 +2153,11 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port) return ap; } +/** Return true iff <a>a</b> and <b>b</b> are the same address and port */ +int +tor_addr_port_eq(const tor_addr_port_t *a, + const tor_addr_port_t *b) +{ + return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port; +} + diff --git a/src/common/address.h b/src/common/address.h index d57abd0d9e..2b9546782e 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -44,7 +44,7 @@ #endif // TODO win32 specific includes -#endif // ADDRESS_PRIVATE +#endif /* defined(ADDRESS_PRIVATE) */ /** The number of bits from an address to consider while doing a masked * comparison. */ @@ -190,7 +190,8 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) */ #define TOR_ADDR_BUF_LEN 48 -int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); +MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family, + tor_addr_t *addr_out)); char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC; /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6 @@ -344,6 +345,8 @@ get_interface_address_list(int severity, int include_internal) } tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port); +int tor_addr_port_eq(const tor_addr_port_t *a, + const tor_addr_port_t *b); #ifdef ADDRESS_PRIVATE MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity, @@ -357,23 +360,23 @@ STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa, sa_family_t family); STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity, sa_family_t family); -#endif +#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */ #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST STATIC smartlist_t *ip_adapter_addresses_to_smartlist( const IP_ADAPTER_ADDRESSES *addresses); STATIC smartlist_t *get_interface_addresses_win32(int severity, sa_family_t family); -#endif +#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */ #ifdef HAVE_IFCONF_TO_SMARTLIST STATIC smartlist_t *ifreq_to_smartlist(char *ifr, size_t buflen); STATIC smartlist_t *get_interface_addresses_ioctl(int severity, sa_family_t family); -#endif +#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ -#endif // ADDRESS_PRIVATE +#endif /* defined(ADDRESS_PRIVATE) */ -#endif +#endif /* !defined(TOR_ADDRESS_H) */ diff --git a/src/common/aes.c b/src/common/aes.c index 35c2d1e3a5..20b51a6758 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -66,11 +66,11 @@ ENABLE_GCC_WARNING(redundant-decls) #elif OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,0,1) && \ (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || \ - defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__)) \ + defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__)) #define USE_EVP_AES_CTR -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,1,0) || ... */ /* We have 2 strategies for getting the AES block cipher: Via OpenSSL's * AES_encrypt function, or via OpenSSL's EVP_EncryptUpdate function. @@ -142,7 +142,7 @@ evaluate_ctr_for_aes(void) { return 0; } -#else +#else /* !(defined(USE_EVP_AES_CTR)) */ /*======================================================================*/ /* Interface to AES code, and counter implementation */ @@ -163,7 +163,7 @@ struct aes_cnt_cipher { uint32_t counter2; uint32_t counter1; uint32_t counter0; -#endif +#endif /* !defined(WORDS_BIGENDIAN) */ union { /** The counter, in big-endian order, as bytes. */ @@ -212,7 +212,7 @@ evaluate_evp_for_aes(int force_val) log_info(LD_CRYPTO, "No AES engine found; using AES_* functions."); should_use_EVP = 0; } -#endif +#endif /* defined(DISABLE_ENGINES) */ return 0; } @@ -312,7 +312,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits) cipher->counter1 = 0; cipher->counter2 = 0; cipher->counter3 = 0; -#endif +#endif /* defined(USING_COUNTER_VARS) */ memset(cipher->ctr_buf.buf, 0, sizeof(cipher->ctr_buf.buf)); @@ -341,7 +341,7 @@ aes_cipher_free(aes_cnt_cipher_t *cipher) STMT_END #else #define UPDATE_CTR_BUF(c, n) -#endif +#endif /* defined(USING_COUNTER_VARS) */ /* Helper function to use EVP with openssl's counter-mode wrapper. */ static void @@ -396,10 +396,10 @@ aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv) cipher->counter2 = ntohl(get_uint32(iv+4)); cipher->counter1 = ntohl(get_uint32(iv+8)); cipher->counter0 = ntohl(get_uint32(iv+12)); -#endif +#endif /* defined(USING_COUNTER_VARS) */ cipher->pos = 0; memcpy(cipher->ctr_buf.buf, iv, 16); } -#endif +#endif /* defined(USE_EVP_AES_CTR) */ diff --git a/src/common/aes.h b/src/common/aes.h index 1cda53f2fa..1e400a56e0 100644 --- a/src/common/aes.h +++ b/src/common/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ @@ -23,5 +23,5 @@ void aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len); int evaluate_evp_for_aes(int force_value); int evaluate_ctr_for_aes(void); -#endif +#endif /* !defined(TOR_AES_H) */ diff --git a/src/common/backtrace.c b/src/common/backtrace.c index 81e04e94eb..f2498b2aa6 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -37,7 +37,7 @@ #include <sys/ucontext.h> #elif defined(HAVE_UCONTEXT_H) #include <ucontext.h> -#endif +#endif /* defined(HAVE_CYGWIN_SIGNAL_H) || ... */ #define EXPOSE_CLEAN_BACKTRACE #include "backtrace.h" @@ -76,21 +76,21 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) #ifdef PC_FROM_UCONTEXT #if defined(__linux__) const size_t n = 1; -#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \ +#elif defined(__darwin__) || defined(__APPLE__) || defined(OpenBSD) \ || defined(__FreeBSD__) const size_t n = 2; #else const size_t n = 1; -#endif +#endif /* defined(__linux__) || ... */ if (depth <= n) return; stack[n] = (void*) ctx->PC_FROM_UCONTEXT; -#else +#else /* !(defined(PC_FROM_UCONTEXT)) */ (void) depth; (void) ctx; (void) stack; -#endif +#endif /* defined(PC_FROM_UCONTEXT) */ } /** Log a message <b>msg</b> at <b>severity</b> in <b>domain</b>, and follow @@ -202,7 +202,7 @@ remove_bt_handler(void) { tor_mutex_uninit(&cb_buf_mutex); } -#endif +#endif /* defined(USE_BACKTRACE) */ #ifdef NO_BACKTRACE_IMPL void @@ -221,7 +221,7 @@ static void remove_bt_handler(void) { } -#endif +#endif /* defined(NO_BACKTRACE_IMPL) */ /** Set up code to handle generating error messages on crashes. */ int diff --git a/src/common/backtrace.h b/src/common/backtrace.h index b53fd2c668..3d0ab8a90a 100644 --- a/src/common/backtrace.h +++ b/src/common/backtrace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BACKTRACE_H @@ -15,7 +15,7 @@ void clean_up_backtrace_handler(void); defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx); #endif -#endif +#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */ -#endif +#endif /* !defined(TOR_BACKTRACE_H) */ diff --git a/src/common/buffers.c b/src/common/buffers.c new file mode 100644 index 0000000000..c45e13d551 --- /dev/null +++ b/src/common/buffers.c @@ -0,0 +1,1077 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.c + * \brief Implements a generic buffer interface. + * + * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush + * from memory, sockets, file descriptors, TLS connections, or another buf_t. + * Buffers are implemented as linked lists of memory chunks. + * + * All socket-backed and TLS-based connection_t objects have a pair of + * buffers: one for incoming data, and one for outcoming data. These are fed + * and drained from functions in connection.c, trigged by events that are + * monitored in main.c. + **/ + +#define BUFFERS_PRIVATE +#include "orconfig.h" +#include <stddef.h> +#include "buffers.h" +#include "compat.h" +#include "compress.h" +#include "util.h" +#include "torint.h" +#include "torlog.h" +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +//#define PARANOIA + +#ifdef PARANOIA +/** Helper: If PARANOIA is defined, assert that the buffer in local variable + * <b>buf</b> is well-formed. */ +#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END +#else +#define check() STMT_NIL +#endif /* defined(PARANOIA) */ + +/* Implementation notes: + * + * After flirting with memmove, and dallying with ring-buffers, we're finally + * getting up to speed with the 1970s and implementing buffers as a linked + * list of small chunks. Each buffer has such a list; data is removed from + * the head of the list, and added at the tail. The list is singly linked, + * and the buffer keeps a pointer to the head and the tail. + * + * Every chunk, except the tail, contains at least one byte of data. Data in + * each chunk is contiguous. + * + * When you need to treat the first N characters on a buffer as a contiguous + * string, use the buf_pullup function to make them so. Don't do this more + * than necessary. + * + * The major free Unix kernels have handled buffers like this since, like, + * forever. + */ + +/* Chunk manipulation functions */ + +#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0]) + +/* We leave this many NUL bytes at the end of the buffer. */ +#ifdef DISABLE_MEMORY_SENTINELS +#define SENTINEL_LEN 0 +#else +#define SENTINEL_LEN 4 +#endif + +/* Header size plus NUL bytes at the end */ +#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN) + +/** Return the number of bytes needed to allocate a chunk to hold + * <b>memlen</b> bytes. */ +#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen)) +/** Return the number of usable bytes in a chunk allocated with + * malloc(<b>memlen</b>). */ +#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD) + +#define DEBUG_SENTINEL + +#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS) +#define DBG_S(s) s +#else +#define DBG_S(s) (void)0 +#endif + +#ifdef DISABLE_MEMORY_SENTINELS +#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL +#else +#define CHUNK_SET_SENTINEL(chunk, alloclen) do { \ + uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen]; \ + DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]); \ + DBG_S(tor_assert(a == b)); \ + memset(a,0,SENTINEL_LEN); \ + } while (0) +#endif /* defined(DISABLE_MEMORY_SENTINELS) */ + +/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem, + * to free up space at the end. */ +static inline void +chunk_repack(chunk_t *chunk) +{ + if (chunk->datalen && chunk->data != &chunk->mem[0]) { + memmove(chunk->mem, chunk->data, chunk->datalen); + } + chunk->data = &chunk->mem[0]; +} + +/** Keep track of total size of allocated chunks for consistency asserts */ +static size_t total_bytes_allocated_in_chunks = 0; +static void +buf_chunk_free_unchecked(chunk_t *chunk) +{ + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + tor_free(chunk); +} +static inline chunk_t * +chunk_new_with_alloc_size(size_t alloc) +{ + chunk_t *ch; + ch = tor_malloc(alloc); + ch->next = NULL; + ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif + ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; + ch->data = &ch->mem[0]; + CHUNK_SET_SENTINEL(ch, alloc); + return ch; +} + +/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a + * new pointer to <b>chunk</b>. Old pointers are no longer valid. */ +static inline chunk_t * +chunk_grow(chunk_t *chunk, size_t sz) +{ + off_t offset; + const size_t memlen_orig = chunk->memlen; + const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); + const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); + tor_assert(sz > chunk->memlen); + offset = chunk->data - chunk->mem; + chunk = tor_realloc(chunk, new_alloc); + chunk->memlen = sz; + chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == orig_alloc); + chunk->DBG_alloc = new_alloc; +#endif + total_bytes_allocated_in_chunks += new_alloc - orig_alloc; + CHUNK_SET_SENTINEL(chunk, new_alloc); + return chunk; +} + +/** Every chunk should take up at least this many bytes. */ +#define MIN_CHUNK_ALLOC 256 +/** No chunk should take up more than this many bytes. */ +#define MAX_CHUNK_ALLOC 65536 + +/** Return the allocation size we'd like to use to hold <b>target</b> + * bytes. */ +size_t +buf_preferred_chunk_size(size_t target) +{ + tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD); + if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC) + return CHUNK_ALLOC_SIZE(target); + size_t sz = MIN_CHUNK_ALLOC; + while (CHUNK_SIZE_WITH_ALLOC(sz) < target) { + sz <<= 1; + } + return sz; +} + +/** Collapse data from the first N chunks from <b>buf</b> into buf->head, + * growing it as necessary, until buf->head has the first <b>bytes</b> bytes + * of data from the buffer, or until buf->head has all the data in <b>buf</b>. + * + * Set *<b>head_out</b> to point to the first byte of available data, and + * *<b>len_out</b> to the number of bytes of data available at + * *<b>head_out</b>. Note that *<b>len_out</b> may be more or less than + * <b>bytes</b>, depending on the number of bytes available. + */ +void +buf_pullup(buf_t *buf, size_t bytes, const char **head_out, size_t *len_out) +{ + chunk_t *dest, *src; + size_t capacity; + if (!buf->head) { + *head_out = NULL; + *len_out = 0; + return; + } + + check(); + if (buf->datalen < bytes) + bytes = buf->datalen; + + capacity = bytes; + if (buf->head->datalen >= bytes) { + *head_out = buf->head->data; + *len_out = buf->head->datalen; + return; + } + + if (buf->head->memlen >= capacity) { + /* We don't need to grow the first chunk, but we might need to repack it.*/ + size_t needed = capacity - buf->head->datalen; + if (CHUNK_REMAINING_CAPACITY(buf->head) < needed) + chunk_repack(buf->head); + tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed); + } else { + chunk_t *newhead; + size_t newsize; + /* We need to grow the chunk. */ + chunk_repack(buf->head); + newsize = CHUNK_SIZE_WITH_ALLOC(buf_preferred_chunk_size(capacity)); + newhead = chunk_grow(buf->head, newsize); + tor_assert(newhead->memlen >= capacity); + if (newhead != buf->head) { + if (buf->tail == buf->head) + buf->tail = newhead; + buf->head = newhead; + } + } + + dest = buf->head; + while (dest->datalen < bytes) { + size_t n = bytes - dest->datalen; + src = dest->next; + tor_assert(src); + if (n >= src->datalen) { + memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen); + dest->datalen += src->datalen; + dest->next = src->next; + if (buf->tail == src) + buf->tail = dest; + buf_chunk_free_unchecked(src); + } else { + memcpy(CHUNK_WRITE_PTR(dest), src->data, n); + dest->datalen += n; + src->data += n; + src->datalen -= n; + tor_assert(dest->datalen == bytes); + } + } + + check(); + *head_out = buf->head->data; + *len_out = buf->head->datalen; +} + +#ifdef TOR_UNIT_TESTS +/* Write sz bytes from cp into a newly allocated buffer buf. + * Returns NULL when passed a NULL cp or zero sz. + * Asserts on failure: only for use in unit tests. + * buf must be freed using buf_free(). */ +buf_t * +buf_new_with_data(const char *cp, size_t sz) +{ + /* Validate arguments */ + if (!cp || sz <= 0) { + return NULL; + } + + tor_assert(sz < SSIZE_T_CEILING); + + /* Allocate a buffer */ + buf_t *buf = buf_new_with_capacity(sz); + tor_assert(buf); + buf_assert_ok(buf); + tor_assert(!buf->head); + + /* Allocate a chunk that is sz bytes long */ + buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz)); + buf->tail = buf->head; + tor_assert(buf->head); + buf_assert_ok(buf); + tor_assert(buf_allocation(buf) >= sz); + + /* Copy the data and size the buffers */ + tor_assert(sz <= buf_slack(buf)); + tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head)); + memcpy(&buf->head->mem[0], cp, sz); + buf->datalen = sz; + buf->head->datalen = sz; + buf->head->data = &buf->head->mem[0]; + buf_assert_ok(buf); + + /* Make sure everything is large enough */ + tor_assert(buf_allocation(buf) >= sz); + tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf)); + /* Does the buffer implementation allocate more than the requested size? + * (for example, by rounding up). If so, these checks will fail. */ + tor_assert(buf_datalen(buf) == sz); + tor_assert(buf_slack(buf) == 0); + + return buf; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** Remove the first <b>n</b> bytes from buf. */ +void +buf_drain(buf_t *buf, size_t n) +{ + tor_assert(buf->datalen >= n); + while (n) { + tor_assert(buf->head); + if (buf->head->datalen > n) { + buf->head->datalen -= n; + buf->head->data += n; + buf->datalen -= n; + return; + } else { + chunk_t *victim = buf->head; + n -= victim->datalen; + buf->datalen -= victim->datalen; + buf->head = victim->next; + if (buf->tail == victim) + buf->tail = NULL; + buf_chunk_free_unchecked(victim); + } + } + check(); +} + +/** Create and return a new buf with default chunk capacity <b>size</b>. + */ +buf_t * +buf_new_with_capacity(size_t size) +{ + buf_t *b = buf_new(); + b->default_chunk_size = buf_preferred_chunk_size(size); + return b; +} + +/** Allocate and return a new buffer with default capacity. */ +buf_t * +buf_new(void) +{ + buf_t *buf = tor_malloc_zero(sizeof(buf_t)); + buf->magic = BUFFER_MAGIC; + buf->default_chunk_size = 4096; + return buf; +} + +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + +/** Remove all data from <b>buf</b>. */ +void +buf_clear(buf_t *buf) +{ + chunk_t *chunk, *next; + buf->datalen = 0; + for (chunk = buf->head; chunk; chunk = next) { + next = chunk->next; + buf_chunk_free_unchecked(chunk); + } + buf->head = buf->tail = NULL; +} + +/** Return the number of bytes stored in <b>buf</b> */ +MOCK_IMPL(size_t, +buf_datalen, (const buf_t *buf)) +{ + return buf->datalen; +} + +/** Return the total length of all chunks used in <b>buf</b>. */ +size_t +buf_allocation(const buf_t *buf) +{ + size_t total = 0; + const chunk_t *chunk; + for (chunk = buf->head; chunk; chunk = chunk->next) { + total += CHUNK_ALLOC_SIZE(chunk->memlen); + } + return total; +} + +/** Return the number of bytes that can be added to <b>buf</b> without + * performing any additional allocation. */ +size_t +buf_slack(const buf_t *buf) +{ + if (!buf->tail) + return 0; + else + return CHUNK_REMAINING_CAPACITY(buf->tail); +} + +/** Release storage held by <b>buf</b>. */ +void +buf_free(buf_t *buf) +{ + if (!buf) + return; + + buf_clear(buf); + buf->magic = 0xdeadbeef; + tor_free(buf); +} + +/** Return a new copy of <b>in_chunk</b> */ +static chunk_t * +chunk_copy(const chunk_t *in_chunk) +{ + chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif + newch->next = NULL; + if (in_chunk->data) { + off_t offset = in_chunk->data - in_chunk->mem; + newch->data = newch->mem + offset; + } + return newch; +} + +/** Return a new copy of <b>buf</b> */ +buf_t * +buf_copy(const buf_t *buf) +{ + chunk_t *ch; + buf_t *out = buf_new(); + out->default_chunk_size = buf->default_chunk_size; + for (ch = buf->head; ch; ch = ch->next) { + chunk_t *newch = chunk_copy(ch); + if (out->tail) { + out->tail->next = newch; + out->tail = newch; + } else { + out->head = out->tail = newch; + } + } + out->datalen = buf->datalen; + return out; +} + +/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to + * the tail of <b>buf</b>. If <b>capped</b>, don't allocate a chunk bigger + * than MAX_CHUNK_ALLOC. */ +chunk_t * +buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) +{ + chunk_t *chunk; + + if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { + chunk = chunk_new_with_alloc_size(buf->default_chunk_size); + } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { + chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC); + } else { + chunk = chunk_new_with_alloc_size(buf_preferred_chunk_size(capacity)); + } + + chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec(); + + if (buf->tail) { + tor_assert(buf->head); + buf->tail->next = chunk; + buf->tail = chunk; + } else { + tor_assert(!buf->head); + buf->head = buf->tail = chunk; + } + check(); + return chunk; +} + +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * milliseconds. Requires the current monotonic time, in truncated msec, + * as its input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + +/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into + * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set + * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, + * and the number of bytes read otherwise. */ +static inline int +read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, + int *reached_eof, int *socket_error) +{ + ssize_t read_result; + if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) + at_most = CHUNK_REMAINING_CAPACITY(chunk); + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (read_result < 0) { + int e = tor_socket_errno(fd); + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ +#ifdef _WIN32 + if (e == WSAENOBUFS) + log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); +#endif + *socket_error = e; + return -1; + } + return 0; /* would block. */ + } else if (read_result == 0) { + log_debug(LD_NET,"Encountered eof on fd %d", (int)fd); + *reached_eof = 1; + return 0; + } else { /* actually got bytes. */ + buf->datalen += read_result; + chunk->datalen += read_result; + log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result, + (int)buf->datalen); + tor_assert(read_result < INT_MAX); + return (int)read_result; + } +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +/* XXXX indicate "read blocked" somehow? */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + /* XXXX It's stupid to overload the return values for these functions: + * "error status" and "number of bytes read" are not mutually exclusive. + */ + int r = 0; + size_t total_read = 0; + + check(); + tor_assert(reached_eof); + tor_assert(SOCKET_OK(s)); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - at_most)) + return -1; + + while (at_most > total_read) { + size_t readlen = at_most - total_read; + chunk_t *chunk; + if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { + chunk = buf_add_chunk_with_capacity(buf, at_most, 1); + if (readlen > chunk->memlen) + readlen = chunk->memlen; + } else { + size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); + chunk = buf->tail; + if (cap < readlen) + readlen = cap; + } + + r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + check(); + if (r < 0) + return r; /* Error */ + tor_assert(total_read+r < INT_MAX); + total_read += r; + if ((size_t)r < readlen) { /* eof, block, or no more to read. */ + break; + } + } + return (int)total_read; +} + +/** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk + * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct + * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes + * written on success, 0 on blocking, -1 on failure. + */ +static inline int +flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen) +{ + ssize_t write_result; + + if (sz > chunk->datalen) + sz = chunk->datalen; + write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (write_result < 0) { + int e = tor_socket_errno(s); + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ +#ifdef _WIN32 + if (e == WSAENOBUFS) + log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?"); +#endif + return -1; + } + log_debug(LD_NET,"write() would block, returning."); + return 0; + } else { + *buf_flushlen -= write_result; + buf_drain(buf, write_result); + tor_assert(write_result < INT_MAX); + return (int)write_result; + } +} + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + /* XXXX It's stupid to overload the return values for these functions: + * "error status" and "number of bytes flushed" are not mutually exclusive. + */ + int r; + size_t flushed = 0; + tor_assert(buf_flushlen); + tor_assert(SOCKET_OK(s)); + if (BUG(*buf_flushlen > buf->datalen)) { + *buf_flushlen = buf->datalen; + } + if (BUG(sz > *buf_flushlen)) { + sz = *buf_flushlen; + } + + check(); + while (sz) { + size_t flushlen0; + tor_assert(buf->head); + if (buf->head->datalen >= sz) + flushlen0 = sz; + else + flushlen0 = buf->head->datalen; + + r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + check(); + if (r < 0) + return r; + flushed += r; + sz -= r; + if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */ + break; + } + tor_assert(flushed < INT_MAX); + return (int)flushed; +} + +/** Append <b>string_len</b> bytes from <b>string</b> to the end of + * <b>buf</b>. + * + * Return the new length of the buffer on success, -1 on failure. + */ +int +buf_add(buf_t *buf, const char *string, size_t string_len) +{ + if (!string_len) + return (int)buf->datalen; + check(); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - string_len)) + return -1; + + while (string_len) { + size_t copy; + if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail)) + buf_add_chunk_with_capacity(buf, string_len, 1); + + copy = CHUNK_REMAINING_CAPACITY(buf->tail); + if (copy > string_len) + copy = string_len; + memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy); + string_len -= copy; + string += copy; + buf->datalen += copy; + buf->tail->datalen += copy; + } + + check(); + tor_assert(buf->datalen < INT_MAX); + return (int)buf->datalen; +} + +/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b> + * onto <b>string</b>. + */ +void +buf_peek(const buf_t *buf, char *string, size_t string_len) +{ + chunk_t *chunk; + + tor_assert(string); + /* make sure we don't ask for too much */ + tor_assert(string_len <= buf->datalen); + /* buf_assert_ok(buf); */ + + chunk = buf->head; + while (string_len) { + size_t copy = string_len; + tor_assert(chunk); + if (chunk->datalen < copy) + copy = chunk->datalen; + memcpy(string, chunk->data, copy); + string_len -= copy; + string += copy; + chunk = chunk->next; + } +} + +/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store + * them into <b>string</b>. Return the new buffer size. <b>string_len</b> + * must be \<= the number of bytes on the buffer. + */ +int +buf_get_bytes(buf_t *buf, char *string, size_t string_len) +{ + /* There must be string_len bytes in buf; write them onto string, + * then memmove buf back (that is, remove them from buf). + * + * Return the number of bytes still on the buffer. */ + + check(); + buf_peek(buf, string, string_len); + buf_drain(buf, string_len); + check(); + tor_assert(buf->datalen < INT_MAX); + return (int)buf->datalen; +} + +/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to + * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately. + * Return the number of bytes actually copied. + */ +int +buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen) +{ + /* We can do way better here, but this doesn't turn up in any profiles. */ + char b[4096]; + size_t cp, len; + + if (BUG(buf_out->datalen >= INT_MAX)) + return -1; + if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen)) + return -1; + + len = *buf_flushlen; + if (len > buf_in->datalen) + len = buf_in->datalen; + + cp = len; /* Remember the number of bytes we intend to copy. */ + tor_assert(cp < INT_MAX); + while (len) { + /* This isn't the most efficient implementation one could imagine, since + * it does two copies instead of 1, but I kinda doubt that this will be + * critical path. */ + size_t n = len > sizeof(b) ? sizeof(b) : len; + buf_get_bytes(buf_in, b, n); + buf_add(buf_out, b, n); + len -= n; + } + *buf_flushlen -= cp; + return (int)cp; +} + +/** Internal structure: represents a position in a buffer. */ +typedef struct buf_pos_t { + const chunk_t *chunk; /**< Which chunk are we pointing to? */ + int pos;/**< Which character inside the chunk's data are we pointing to? */ + size_t chunk_pos; /**< Total length of all previous chunks. */ +} buf_pos_t; + +/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/ +static void +buf_pos_init(const buf_t *buf, buf_pos_t *out) +{ + out->chunk = buf->head; + out->pos = 0; + out->chunk_pos = 0; +} + +/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current + * position of <b>out</b>, or later. Return -1 if no instances are found; + * otherwise returns the absolute position of the character. */ +static off_t +buf_find_pos_of_char(char ch, buf_pos_t *out) +{ + const chunk_t *chunk; + int pos; + tor_assert(out); + if (out->chunk) { + if (out->chunk->datalen) { + tor_assert(out->pos < (off_t)out->chunk->datalen); + } else { + tor_assert(out->pos == 0); + } + } + pos = out->pos; + for (chunk = out->chunk; chunk; chunk = chunk->next) { + char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos); + if (cp) { + out->chunk = chunk; + tor_assert(cp - chunk->data < INT_MAX); + out->pos = (int)(cp - chunk->data); + return out->chunk_pos + out->pos; + } else { + out->chunk_pos += chunk->datalen; + pos = 0; + } + } + return -1; +} + +/** Advance <b>pos</b> by a single character, if there are any more characters + * in the buffer. Returns 0 on success, -1 on failure. */ +static inline int +buf_pos_inc(buf_pos_t *pos) +{ + ++pos->pos; + if (pos->pos == (off_t)pos->chunk->datalen) { + if (!pos->chunk->next) + return -1; + pos->chunk_pos += pos->chunk->datalen; + pos->chunk = pos->chunk->next; + pos->pos = 0; + } + return 0; +} + +/** Return true iff the <b>n</b>-character string in <b>s</b> appears + * (verbatim) at <b>pos</b>. */ +static int +buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n) +{ + buf_pos_t p; + if (!n) + return 1; + + memcpy(&p, pos, sizeof(p)); + + while (1) { + char ch = p.chunk->data[p.pos]; + if (ch != *s) + return 0; + ++s; + /* If we're out of characters that don't match, we match. Check this + * _before_ we test incrementing pos, in case we're at the end of the + * string. */ + if (--n == 0) + return 1; + if (buf_pos_inc(&p)<0) + return 0; + } +} + +/** Return the first position in <b>buf</b> at which the <b>n</b>-character + * string <b>s</b> occurs, or -1 if it does not occur. */ +int +buf_find_string_offset(const buf_t *buf, const char *s, size_t n) +{ + buf_pos_t pos; + buf_pos_init(buf, &pos); + while (buf_find_pos_of_char(*s, &pos) >= 0) { + if (buf_matches_at_pos(&pos, s, n)) { + tor_assert(pos.chunk_pos + pos.pos < INT_MAX); + return (int)(pos.chunk_pos + pos.pos); + } else { + if (buf_pos_inc(&pos)<0) + return -1; + } + } + return -1; +} + +/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null + * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */ +int +buf_peek_startswith(const buf_t *buf, const char *cmd) +{ + char tmp[PEEK_BUF_STARTSWITH_MAX]; + size_t clen = strlen(cmd); + if (clen == 0) + return 1; + if (BUG(clen > sizeof(tmp))) + return 0; + if (buf->datalen < clen) + return 0; + buf_peek(buf, tmp, clen); + return fast_memeq(tmp, cmd, clen); +} + +/** Return the index within <b>buf</b> at which <b>ch</b> first appears, + * or -1 if <b>ch</b> does not appear on buf. */ +static off_t +buf_find_offset_of_char(buf_t *buf, char ch) +{ + chunk_t *chunk; + off_t offset = 0; + for (chunk = buf->head; chunk; chunk = chunk->next) { + char *cp = memchr(chunk->data, ch, chunk->datalen); + if (cp) + return offset + (cp - chunk->data); + else + offset += chunk->datalen; + } + return -1; +} + +/** Try to read a single LF-terminated line from <b>buf</b>, and write it + * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer + * at <b>data_out</b>. Set *<b>data_len</b> to the number of bytes in the + * line, not counting the terminating NUL. Return 1 if we read a whole line, + * return 0 if we don't have a whole line yet, and return -1 if the line + * length exceeds *<b>data_len</b>. + */ +int +buf_get_line(buf_t *buf, char *data_out, size_t *data_len) +{ + size_t sz; + off_t offset; + + if (!buf->head) + return 0; + + offset = buf_find_offset_of_char(buf, '\n'); + if (offset < 0) + return 0; + sz = (size_t) offset; + if (sz+2 > *data_len) { + *data_len = sz + 2; + return -1; + } + buf_get_bytes(buf, data_out, sz+1); + data_out[sz+1] = '\0'; + *data_len = sz+1; + return 1; +} + +/** Compress or uncompress the <b>data_len</b> bytes in <b>data</b> using the + * compression state <b>state</b>, appending the result to <b>buf</b>. If + * <b>done</b> is true, flush the data in the state and finish the + * compression/uncompression. Return -1 on failure, 0 on success. */ +int +buf_add_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + const int done) +{ + char *next; + size_t old_avail, avail; + int over = 0; + + do { + int need_new_chunk = 0; + if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) { + size_t cap = data_len / 4; + buf_add_chunk_with_capacity(buf, cap, 1); + } + next = CHUNK_WRITE_PTR(buf->tail); + avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: + over = 1; + break; + case TOR_COMPRESS_ERROR: + return -1; + case TOR_COMPRESS_OK: + if (data_len == 0) { + tor_assert_nonfatal(!done); + over = 1; + } + break; + case TOR_COMPRESS_BUFFER_FULL: + if (avail) { + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ + need_new_chunk = 1; + } + if (data_len == 0 && !done) { + /* We've consumed all the input data, though, so there's no + * point in forging ahead right now. */ + over = 1; + } + break; + } + buf->datalen += old_avail - avail; + buf->tail->datalen += old_avail - avail; + if (need_new_chunk) { + buf_add_chunk_with_capacity(buf, data_len/4, 1); + } + + } while (!over); + check(); + return 0; +} + +/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */ +int +buf_set_to_copy(buf_t **output, + const buf_t *input) +{ + if (*output) + buf_free(*output); + *output = buf_copy(input); + return 0; +} + +/** Log an error and exit if <b>buf</b> is corrupted. + */ +void +buf_assert_ok(buf_t *buf) +{ + tor_assert(buf); + tor_assert(buf->magic == BUFFER_MAGIC); + + if (! buf->head) { + tor_assert(!buf->tail); + tor_assert(buf->datalen == 0); + } else { + chunk_t *ch; + size_t total = 0; + tor_assert(buf->tail); + for (ch = buf->head; ch; ch = ch->next) { + total += ch->datalen; + tor_assert(ch->datalen <= ch->memlen); + tor_assert(ch->data >= &ch->mem[0]); + tor_assert(ch->data <= &ch->mem[0]+ch->memlen); + if (ch->data == &ch->mem[0]+ch->memlen) { + /* LCOV_EXCL_START */ + static int warned = 0; + if (! warned) { + log_warn(LD_BUG, "Invariant violation in buf.c related to #15083"); + warned = 1; + } + /* LCOV_EXCL_STOP */ + } + tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen); + if (!ch->next) + tor_assert(ch == buf->tail); + } + tor_assert(buf->datalen == total); + } +} + diff --git a/src/common/buffers.h b/src/common/buffers.h new file mode 100644 index 0000000000..1eaa5f2d04 --- /dev/null +++ b/src/common/buffers.h @@ -0,0 +1,125 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.h + * \brief Header file for buffers.c. + **/ + +#ifndef TOR_BUFFERS_H +#define TOR_BUFFERS_H + +#include "compat.h" +#include "compat.h" +#include "torint.h" +#include "testsupport.h" + +typedef struct buf_t buf_t; + +struct tor_compress_state_t; + +buf_t *buf_new(void); +buf_t *buf_new_with_capacity(size_t size); +size_t buf_get_default_chunk_size(const buf_t *buf); +void buf_free(buf_t *buf); +void buf_clear(buf_t *buf); +buf_t *buf_copy(const buf_t *buf); + +MOCK_DECL(size_t, buf_datalen, (const buf_t *buf)); +size_t buf_allocation(const buf_t *buf); +size_t buf_slack(const buf_t *buf); + +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + +int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen); + +int buf_add(buf_t *buf, const char *string, size_t string_len); +int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state, + const char *data, size_t data_len, int done); +int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); +void buf_peek(const buf_t *buf, char *string, size_t string_len); +void buf_drain(buf_t *buf, size_t n); +int buf_get_bytes(buf_t *buf, char *string, size_t string_len); +int buf_get_line(buf_t *buf, char *data_out, size_t *data_len); + +#define PEEK_BUF_STARTSWITH_MAX 16 +int buf_peek_startswith(const buf_t *buf, const char *cmd); + +int buf_set_to_copy(buf_t **output, + const buf_t *input); + +void buf_assert_ok(buf_t *buf); + +int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +void buf_pullup(buf_t *buf, size_t bytes, + const char **head_out, size_t *len_out); + +#ifdef BUFFERS_PRIVATE +#ifdef TOR_UNIT_TESTS +buf_t *buf_new_with_data(const char *cp, size_t sz); +#endif +size_t buf_preferred_chunk_size(size_t target); + +#define DEBUG_CHUNK_ALLOC +/** A single chunk on a buffer. */ +typedef struct chunk_t { + struct chunk_t *next; /**< The next chunk on the buffer. */ + size_t datalen; /**< The number of bytes stored in this chunk */ + size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ +#ifdef DEBUG_CHUNK_ALLOC + size_t DBG_alloc; +#endif + char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ + uint32_t inserted_time; /**< Timestamp in truncated ms since epoch + * when this chunk was inserted. */ + char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in + * this chunk. */ +} chunk_t; + +/** Magic value for buf_t.magic, to catch pointer errors. */ +#define BUFFER_MAGIC 0xB0FFF312u +/** A resizeable buffer, optimized for reading and writing. */ +struct buf_t { + uint32_t magic; /**< Magic cookie for debugging: Must be set to + * BUFFER_MAGIC. */ + size_t datalen; /**< How many bytes is this buffer holding right now? */ + size_t default_chunk_size; /**< Don't allocate any chunks smaller than + * this for this buffer. */ + chunk_t *head; /**< First chunk in the list, or NULL for none. */ + chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ +}; + +chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped); +/** If a read onto the end of a chunk would be smaller than this number, then + * just start a new chunk. */ +#define MIN_READ_LEN 8 + +/** Return the number of bytes that can be written onto <b>chunk</b> without + * running out of space. */ +static inline size_t +CHUNK_REMAINING_CAPACITY(const chunk_t *chunk) +{ + return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen); +} + +/** Return the next character in <b>chunk</b> onto which data can be appended. + * If the chunk is full, this might be off the end of chunk->mem. */ +static inline char * +CHUNK_WRITE_PTR(chunk_t *chunk) +{ + return chunk->data + chunk->datalen; +} + +#endif /* defined(BUFFERS_PRIVATE) */ + +#endif /* !defined(TOR_BUFFERS_H) */ + diff --git a/src/common/buffers_tls.c b/src/common/buffers_tls.c new file mode 100644 index 0000000000..041f78b818 --- /dev/null +++ b/src/common/buffers_tls.c @@ -0,0 +1,179 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define BUFFERS_PRIVATE +#include "orconfig.h" +#include <stddef.h> +#include "buffers.h" +#include "buffers_tls.h" +#include "compat.h" +#include "compress.h" +#include "util.h" +#include "torint.h" +#include "torlog.h" +#include "tortls.h" +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +/** As read_to_chunk(), but return (negative) error code on error, blocking, + * or TLS, and the number of bytes read otherwise. */ +static inline int +read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, + size_t at_most) +{ + int read_result; + + tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most); + read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most); + if (read_result < 0) + return read_result; + buf->datalen += read_result; + chunk->datalen += read_result; + return read_result; +} + +/** As read_to_buf, but reads from a TLS connection, and returns a TLS + * status value rather than the number of bytes read. + * + * Using TLS on OR connections complicates matters in two ways. + * + * First, a TLS stream has its own read buffer independent of the + * connection's read buffer. (TLS needs to read an entire frame from + * the network before it can decrypt any data. Thus, trying to read 1 + * byte from TLS can require that several KB be read from the network + * and decrypted. The extra data is stored in TLS's decrypt buffer.) + * Because the data hasn't been read by Tor (it's still inside the TLS), + * this means that sometimes a connection "has stuff to read" even when + * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is + * used in connection.c to detect TLS objects with non-empty internal + * buffers and read from them again. + * + * Second, the TLS stream's events do not correspond directly to network + * events: sometimes, before a TLS stream can read, the network must be + * ready to write -- or vice versa. + */ +int +buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most) +{ + int r = 0; + size_t total_read = 0; + + check_no_tls_errors(); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - at_most)) + return -1; + + while (at_most > total_read) { + size_t readlen = at_most - total_read; + chunk_t *chunk; + if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { + chunk = buf_add_chunk_with_capacity(buf, at_most, 1); + if (readlen > chunk->memlen) + readlen = chunk->memlen; + } else { + size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); + chunk = buf->tail; + if (cap < readlen) + readlen = cap; + } + + r = read_to_chunk_tls(buf, chunk, tls, readlen); + if (r < 0) + return r; /* Error */ + tor_assert(total_read+r < INT_MAX); + total_read += r; + if ((size_t)r < readlen) /* eof, block, or no more to read. */ + break; + } + return (int)total_read; +} + +/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk + * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. (Tries to write + * more if there is a forced pending write size.) On success, deduct the + * bytes written from *<b>buf_flushlen</b>. Return the number of bytes + * written on success, and a TOR_TLS error code on failure or blocking. + */ +static inline int +flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, + size_t sz, size_t *buf_flushlen) +{ + int r; + size_t forced; + char *data; + + forced = tor_tls_get_forced_write_size(tls); + if (forced > sz) + sz = forced; + if (chunk) { + data = chunk->data; + tor_assert(sz <= chunk->datalen); + } else { + data = NULL; + tor_assert(sz == 0); + } + r = tor_tls_write(tls, data, sz); + if (r < 0) + return r; + if (*buf_flushlen > (size_t)r) + *buf_flushlen -= r; + else + *buf_flushlen = 0; + buf_drain(buf, r); + log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.", + r,(int)*buf_flushlen,(int)buf->datalen); + return r; +} + +/** As buf_flush_to_socket(), but writes data to a TLS connection. Can write + * more than <b>flushlen</b> bytes. + */ +int +buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen, + size_t *buf_flushlen) +{ + int r; + size_t flushed = 0; + ssize_t sz; + tor_assert(buf_flushlen); + if (BUG(*buf_flushlen > buf->datalen)) { + *buf_flushlen = buf->datalen; + } + if (BUG(flushlen > *buf_flushlen)) { + flushlen = *buf_flushlen; + } + sz = (ssize_t) flushlen; + + /* we want to let tls write even if flushlen is zero, because it might + * have a partial record pending */ + check_no_tls_errors(); + + do { + size_t flushlen0; + if (buf->head) { + if ((ssize_t)buf->head->datalen >= sz) + flushlen0 = sz; + else + flushlen0 = buf->head->datalen; + } else { + flushlen0 = 0; + } + + r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen); + if (r < 0) + return r; + flushed += r; + sz -= r; + if (r == 0) /* Can't flush any more now. */ + break; + } while (sz > 0); + tor_assert(flushed < INT_MAX); + return (int)flushed; +} + diff --git a/src/common/buffers_tls.h b/src/common/buffers_tls.h new file mode 100644 index 0000000000..2f9fda45a0 --- /dev/null +++ b/src/common/buffers_tls.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_BUFFERS_TLS_H +#define TOR_BUFFERS_TLS_H + +struct buf_t; +struct tor_tls_t; + +int buf_read_from_tls(struct buf_t *buf, + struct tor_tls_t *tls, size_t at_most); +int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls, + size_t sz, size_t *buf_flushlen); + +#endif /* !defined(TOR_BUFFERS_TLS_H) */ + diff --git a/src/common/ciphers.inc b/src/common/ciphers.inc index 23f5fd2da4..0084b3e325 100644 --- a/src/common/ciphers.inc +++ b/src/common/ciphers.inc @@ -33,6 +33,26 @@ #else XCIPHER(0xc02f, TLS1_TXT_ECDHE_RSA_WITH_AES_128_GCM_SHA256) #endif +#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + CIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305) +#else + XCIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305) +#endif +#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 + CIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305) +#else + XCIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305) +#endif +#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + CIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) +#else + XCIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) +#endif +#ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + CIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384) +#else + XCIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384) +#endif #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) #else @@ -53,88 +73,28 @@ #else XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA) #endif -#ifdef TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA - CIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA) -#else - XCIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA) -#endif -#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA - CIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA) -#else - XCIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA) -#endif -#ifdef TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA - CIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA) -#else - XCIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA) -#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA) #else XCIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA) #endif -#ifdef TLS1_TXT_DHE_DSS_WITH_AES_128_SHA - CIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA) -#else - XCIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA) -#endif -#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA - CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA) -#else - XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA) -#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) #else XCIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) #endif -#ifdef TLS1_TXT_DHE_DSS_WITH_AES_256_SHA - CIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA) -#else - XCIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA) -#endif -#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA - CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA) -#else - XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA) -#endif -#ifdef SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA - CIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -#else - XCIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -#endif #ifdef TLS1_TXT_RSA_WITH_AES_128_SHA CIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA) #else XCIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA) #endif -#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA - CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA) -#else - XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA) -#endif #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA) #else XCIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA) #endif -#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA - CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA) -#else - XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA) -#endif #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA) #else XCIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA) #endif -#ifdef SSL3_TXT_RSA_RC4_128_SHA - CIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA) -#else - XCIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA) -#endif -#ifdef SSL3_TXT_RSA_RC4_128_MD5 - CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5) -#else - XCIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5) -#endif diff --git a/src/common/compat.c b/src/common/compat.c index 4ac443c134..83bb707e17 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -88,12 +88,12 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) while (cnt--) *vcptr++ = 0; } -#endif +#endif /* defined(HAVE_DECL_SECUREZEROMEMORY) && !HAVE_DECL_SECUREZEROMEMORY */ #elif defined(HAVE_READPASSPHRASE_H) #include <readpassphrase.h> #else #include "tor_readpassphrase.h" -#endif +#endif /* defined(_WIN32) || ... */ /* Includes for the process attaching prevention */ #if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) @@ -102,7 +102,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(__APPLE__) #include <sys/types.h> #include <sys/ptrace.h> -#endif +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) || ... */ #ifdef HAVE_NETDB_H #include <netdb.h> @@ -161,7 +161,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode) * are running on one without. */ if (errno != EINVAL) return -1; -#endif +#endif /* defined(O_CLOEXEC) */ log_debug(LD_FS, "Opening %s with flags %x", p, flags); fd = open(p, flags, mode); @@ -173,7 +173,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode) return -1; } } -#endif +#endif /* defined(FD_CLOEXEC) */ return fd; } @@ -191,7 +191,7 @@ tor_fopen_cloexec(const char *path, const char *mode) return NULL; } } -#endif +#endif /* defined(FD_CLOEXEC) */ return result; } @@ -204,7 +204,16 @@ tor_rename(const char *path_old, const char *path_new) sandbox_intern_string(path_new)); } -#if defined(HAVE_SYS_MMAN_H) || defined(RUNNING_DOXYGEN) +/* Some MinGW builds have sys/mman.h, but not the corresponding symbols. + * Other configs rename the symbols using macros (including getpagesize). + * So check for sys/mman.h and unistd.h, and a getpagesize declaration. */ +#if (defined(HAVE_SYS_MMAN_H) && defined(HAVE_UNISTD_H) && \ + defined(HAVE_DECL_GETPAGESIZE)) +#define COMPAT_HAS_MMAN_AND_PAGESIZE +#endif + +#if defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || \ + defined(RUNNING_DOXYGEN) /** Try to create a memory mapping for <b>filename</b> and return it. On * failure, return NULL. Sets errno properly, using ERANGE to mean * "empty file". */ @@ -250,6 +259,12 @@ tor_mmap_file(const char *filename) page_size = getpagesize(); size += (size%page_size) ? page_size-(size%page_size) : 0; + if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) { + log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename); + errno = EFBIG; + close(fd); + return NULL; + } if (!size) { /* Zero-length file. If we call mmap on it, it will succeed but * return NULL, and bad things will happen. So just fail. */ @@ -436,7 +451,7 @@ tor_munmap_file(tor_mmap_t *handle) /* Can't fail in this mmap()/munmap()-free case */ return 0; } -#endif +#endif /* defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || ... || ... */ /** Replacement for snprintf. Differs from platform snprintf in two * ways: First, always NUL-terminates its output. Second, always @@ -576,7 +591,7 @@ tor_vasprintf(char **strp, const char *fmt, va_list args) } *strp = strp_tmp; return len; -#endif +#endif /* defined(HAVE_VASPRINTF) || ... */ } /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at @@ -622,7 +637,7 @@ tor_memmem(const void *_haystack, size_t hlen, } } return NULL; -#endif +#endif /* defined(HAVE_MEMMEM) && (!defined(__GNUC__) || __GNUC__ >= 2) */ } /** @@ -760,7 +775,7 @@ tor_fix_source_file(const char *fname) } return r; } -#endif +#endif /* defined(_WIN32) */ /** * Read a 16-bit value beginning at <b>cp</b>. Equivalent to @@ -854,7 +869,7 @@ replace_file(const char *from, const char *to) return -1; } return tor_rename(from,to); -#endif +#endif /* !defined(_WIN32) */ } /** Change <b>fname</b>'s modification time to now. */ @@ -940,7 +955,7 @@ tor_lockfile_lock(const char *filename, int blocking, int *locked_out) return NULL; } } -#endif +#endif /* defined(_WIN32) || ... */ result = tor_malloc(sizeof(tor_lockfile_t)); result->filename = tor_strdup(filename); @@ -968,7 +983,7 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) } #else /* Closing the lockfile is sufficient. */ -#endif +#endif /* defined(_WIN32) || ... */ close(lockfile->fd); lockfile->fd = -1; @@ -1016,9 +1031,9 @@ tor_fd_seekend(int fd) * no need to worry. */ if (rc < 0 && errno == ESPIPE) rc = 0; -#endif +#endif /* defined(ESPIPE) */ return (rc < 0) ? -1 : 0; -#endif +#endif /* defined(_WIN32) */ } /** Move <b>fd</b> to position <b>pos</b> in the file. Return -1 on error, 0 @@ -1057,7 +1072,7 @@ tor_ftruncate(int fd) static bitarray_t *open_sockets = NULL; /** The size of <b>open_sockets</b>, in bits. */ static int max_socket = -1; -#endif +#endif /* defined(DEBUG_SOCKET_COUNTING) */ /** Count of number of sockets currently open. (Undercounts sockets opened by * eventdns and libevent.) */ @@ -1127,7 +1142,7 @@ tor_close_socket,(tor_socket_t s)) tor_assert(open_sockets && s <= max_socket); bitarray_clear(open_sockets, s); } -#endif +#endif /* defined(DEBUG_SOCKET_COUNTING) */ if (r == 0) { --n_sockets_open; } else { @@ -1137,7 +1152,7 @@ tor_close_socket,(tor_socket_t s)) #else if (r != EBADF) --n_sockets_open; // LCOV_EXCL_LINE -- EIO and EINTR too hard to force. -#endif +#endif /* defined(_WIN32) */ r = -1; } @@ -1170,9 +1185,9 @@ mark_socket_open(tor_socket_t s) } bitarray_set(open_sockets, s); } -#else +#else /* !(defined(DEBUG_SOCKET_COUNTING)) */ #define mark_socket_open(s) STMT_NIL -#endif +#endif /* defined(DEBUG_SOCKET_COUNTING) */ /** @} */ /** As socket(), but counts the number of open sockets. */ @@ -1230,7 +1245,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, * support, we are running on one without. */ if (errno != EINVAL) return s; -#endif /* SOCK_CLOEXEC && SOCK_NONBLOCK */ +#endif /* defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) */ s = socket(domain, type, protocol); if (! SOCKET_OK(s)) @@ -1244,9 +1259,9 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, return TOR_INVALID_SOCKET; } } -#else +#else /* !(defined(FD_CLOEXEC)) */ (void)cloexec; -#endif +#endif /* defined(FD_CLOEXEC) */ if (nonblock) { if (set_socket_nonblocking(s) == -1) { @@ -1302,7 +1317,8 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } -#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) +#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) \ + && defined(SOCK_NONBLOCK) int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) | (nonblock ? SOCK_NONBLOCK : 0); s = accept4(sockfd, addr, len, ext_flags); @@ -1314,7 +1330,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, * we are missing SOCK_CLOEXEC/SOCK_NONBLOCK support. */ if (errno != EINVAL && errno != ENOSYS) return s; -#endif +#endif /* defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) ... */ s = accept(sockfd, addr, len); if (!SOCKET_OK(s)) @@ -1328,9 +1344,9 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } } -#else +#else /* !(defined(FD_CLOEXEC)) */ (void)cloexec; -#endif +#endif /* defined(FD_CLOEXEC) */ if (nonblock) { if (set_socket_nonblocking(s) == -1) { @@ -1390,7 +1406,7 @@ set_socket_nonblocking(tor_socket_t sock) log_warn(LD_NET, "Couldn't set file status flags: %s", strerror(errno)); return -1; } -#endif +#endif /* defined(_WIN32) */ return 0; } @@ -1428,7 +1444,7 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) * are running on one without. */ if (errno != EINVAL) return -errno; -#endif +#endif /* defined(SOCK_CLOEXEC) */ r = socketpair(family, type, protocol, fd); if (r < 0) @@ -1451,7 +1467,7 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) return -errno; } } -#endif +#endif /* defined(FD_CLOEXEC) */ goto sockets_ok; /* So that sockets_ok will not be unused. */ sockets_ok: @@ -1467,9 +1483,9 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) socket_accounting_unlock(); return 0; -#else +#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */ return tor_ersatz_socketpair(family, type, protocol, fd); -#endif +#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */ } #ifdef NEED_ERSATZ_SOCKETPAIR @@ -1626,7 +1642,7 @@ tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) #undef SIZEOF_SOCKADDR -#endif +#endif /* defined(NEED_ERSATZ_SOCKETPAIR) */ /* Return the maximum number of allowed sockets. */ int @@ -1680,7 +1696,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) #else const char *platform = "unknown platforms with no getrlimit()"; const unsigned long MAX_CONNECTIONS = 15000; -#endif +#endif /* defined(CYGWIN) || defined(__CYGWIN__) || ... */ log_fn(LOG_INFO, LD_NET, "This platform is missing getrlimit(). Proceeding."); if (limit > MAX_CONNECTIONS) { @@ -1691,7 +1707,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) return -1; } limit = MAX_CONNECTIONS; -#else /* HAVE_GETRLIMIT */ +#else /* !(!defined(HAVE_GETRLIMIT)) */ struct rlimit rlim; if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { @@ -1741,7 +1757,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) couldnt_set = 0; } } -#endif /* OPEN_MAX */ +#endif /* defined(OPEN_MAX) */ if (couldnt_set) { log_warn(LD_CONFIG,"Couldn't set maximum number of file descriptors: %s", strerror(setrlimit_errno)); @@ -1749,7 +1765,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) } /* leave some overhead for logs, etc, */ limit = rlim.rlim_cur; -#endif /* HAVE_GETRLIMIT */ +#endif /* !defined(HAVE_GETRLIMIT) */ if (limit > INT_MAX) limit = INT_MAX; @@ -1787,7 +1803,7 @@ log_credential_status(void) "UID is %u (real), %u (effective), %u (saved)", (unsigned)ruid, (unsigned)euid, (unsigned)suid); } -#else +#else /* !(defined(HAVE_GETRESUID)) */ /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */ ruid = getuid(); euid = geteuid(); @@ -1796,7 +1812,7 @@ log_credential_status(void) log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "UID is %u (real), %u (effective), unknown (saved)", (unsigned)ruid, (unsigned)euid); -#endif +#endif /* defined(HAVE_GETRESUID) */ /* log GIDs */ #ifdef HAVE_GETRESGID @@ -1808,7 +1824,7 @@ log_credential_status(void) "GID is %u (real), %u (effective), %u (saved)", (unsigned)rgid, (unsigned)egid, (unsigned)sgid); } -#else +#else /* !(defined(HAVE_GETRESGID)) */ /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */ rgid = getgid(); egid = getegid(); @@ -1816,7 +1832,7 @@ log_credential_status(void) log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "GID is %u (real), %u (effective), unknown (saved)", (unsigned)rgid, (unsigned)egid); -#endif +#endif /* defined(HAVE_GETRESGID) */ /* log supplementary groups */ sup_gids_size = 64; @@ -1856,7 +1872,7 @@ log_credential_status(void) return 0; } -#endif +#endif /* !defined(_WIN32) */ #ifndef _WIN32 /** Cached struct from the last getpwname() call we did successfully. */ @@ -1956,7 +1972,7 @@ tor_getpwuid(uid_t uid) return NULL; } -#endif +#endif /* !defined(_WIN32) */ /** Return true iff we were compiled with capability support, and capabilities * seem to work. **/ @@ -1969,9 +1985,9 @@ have_capability_support(void) return 0; cap_free(caps); return 1; -#else +#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ return 0; -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ } #ifdef HAVE_LINUX_CAPABILITIES @@ -2030,7 +2046,7 @@ drop_capabilities(int pre_setuid) return 0; } -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ /** Call setuid and setgid to run as <b>user</b> and switch to their * primary group. Return 0 on success. On failure, log and return -1. @@ -2080,13 +2096,13 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(1)) return -1; } -#else +#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ (void) keep_bindlow; if (warn_if_no_caps) { log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support " "on this system."); } -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ /* Properly switch egid,gid,euid,uid here or bail out */ if (setgroups(1, &pw->pw_gid)) { @@ -2146,7 +2162,7 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(0)) return -1; } -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ #if !defined(CYGWIN) && !defined(__CYGWIN__) /* If we tried to drop privilege to a group/user other than root, attempt to @@ -2170,7 +2186,7 @@ switch_id(const char *user, const unsigned flags) return -1; } } -#endif +#endif /* !defined(CYGWIN) && !defined(__CYGWIN__) */ /* Check what really happened */ if (log_credential_status()) { @@ -2179,8 +2195,8 @@ switch_id(const char *user, const unsigned flags) have_already_switched_id = 1; /* mark success so we never try again */ -#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL) -#ifdef PR_SET_DUMPABLE +#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && \ + defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE) if (pw->pw_uid) { /* Re-enable core dumps if we're not running as root. */ log_info(LD_CONFIG, "Re-enabling coredumps"); @@ -2188,17 +2204,16 @@ switch_id(const char *user, const unsigned flags) log_warn(LD_CONFIG, "Unable to re-enable coredumps: %s",strerror(errno)); } } -#endif -#endif +#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */ return 0; -#else +#else /* !(!defined(_WIN32)) */ (void)user; (void)flags; log_warn(LD_CONFIG, "Switching users is unsupported on your OS."); return -1; -#endif +#endif /* !defined(_WIN32) */ } /* We only use the linux prctl for now. There is no Win32 support; this may @@ -2221,35 +2236,32 @@ switch_id(const char *user, const unsigned flags) int tor_disable_debugger_attach(void) { - int r, attempted; - r = -1; - attempted = 0; + int r = -1; log_debug(LD_CONFIG, "Attemping to disable debugger attachment to Tor for " "unprivileged users."); -#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL) -#ifdef PR_SET_DUMPABLE - attempted = 1; +#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) \ + && defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE) +#define TRIED_TO_DISABLE r = prctl(PR_SET_DUMPABLE, 0); -#endif -#endif -#if defined(__APPLE__) && defined(PT_DENY_ATTACH) - if (r < 0) { - attempted = 1; - r = ptrace(PT_DENY_ATTACH, 0, 0, 0); - } -#endif +#elif defined(__APPLE__) && defined(PT_DENY_ATTACH) +#define TRIED_TO_ATTACH + r = ptrace(PT_DENY_ATTACH, 0, 0, 0); +#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) ... || ... */ // XXX: TODO - Mac OS X has dtrace and this may be disabled. // XXX: TODO - Windows probably has something similar - if (r == 0 && attempted) { +#ifdef TRIED_TO_DISABLE + if (r == 0) { log_debug(LD_CONFIG,"Debugger attachment disabled for " "unprivileged users."); return 1; - } else if (attempted) { + } else { log_warn(LD_CONFIG, "Unable to disable debugger attaching: %s", strerror(errno)); } +#endif /* defined(TRIED_TO_DISABLE) */ +#undef TRIED_TO_DISABLE return r; } @@ -2268,7 +2280,7 @@ get_user_homedir(const char *username) } return tor_strdup(pw->pw_dir); } -#endif +#endif /* defined(HAVE_PWD_H) */ /** Modify <b>fname</b> to contain the name of its parent directory. Doesn't * actually examine the filesystem; does a purely syntactic modification. @@ -2296,7 +2308,7 @@ get_parent_directory(char *fname) if (fname[0] && fname[1] == ':') { fname += 2; } -#endif +#endif /* defined(_WIN32) */ /* Now we want to remove all path-separators at the end of the string, * and to remove the end of the string starting with the path separator * before the last non-path-separator. In perl, this would be @@ -2335,17 +2347,36 @@ get_parent_directory(char *fname) static char * alloc_getcwd(void) { -#ifdef PATH_MAX -#define MAX_CWD PATH_MAX -#else -#define MAX_CWD 4096 -#endif +#ifdef HAVE_GET_CURRENT_DIR_NAME + /* Glibc makes this nice and simple for us. */ + char *cwd = get_current_dir_name(); + char *result = NULL; + if (cwd) { + /* We make a copy here, in case tor_malloc() is not malloc(). */ + result = tor_strdup(cwd); + raw_free(cwd); // alias for free to avoid tripping check-spaces. + } + return result; +#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */ + size_t size = 1024; + char *buf = NULL; + char *ptr = NULL; + + while (ptr == NULL) { + buf = tor_realloc(buf, size); + ptr = getcwd(buf, size); - char path_buf[MAX_CWD]; - char *path = getcwd(path_buf, sizeof(path_buf)); - return path ? tor_strdup(path) : NULL; + if (ptr == NULL && errno != ERANGE) { + tor_free(buf); + return NULL; + } + + size *= 2; + } + return buf; +#endif /* defined(HAVE_GET_CURRENT_DIR_NAME) */ } -#endif +#endif /* !defined(_WIN32) */ /** Expand possibly relative path <b>fname</b> to an absolute path. * Return a newly allocated string, possibly equal to <b>fname</b>. */ @@ -2361,7 +2392,7 @@ make_path_absolute(char *fname) if (absfname_malloced) raw_free(absfname_malloced); return absfname; -#else +#else /* !(defined(_WIN32)) */ char *absfname = NULL, *path = NULL; tor_assert(fname); @@ -2384,7 +2415,7 @@ make_path_absolute(char *fname) } } return absfname; -#endif +#endif /* defined(_WIN32) */ } #ifndef HAVE__NSGETENVIRON @@ -2393,8 +2424,8 @@ make_path_absolute(char *fname) #ifndef RUNNING_DOXYGEN extern char **environ; #endif -#endif -#endif +#endif /* !defined(HAVE_EXTERN_ENVIRON_DECLARED) */ +#endif /* !defined(HAVE__NSGETENVIRON) */ /** Return the current environment. This is a portable replacement for * 'environ'. */ @@ -2406,9 +2437,9 @@ get_environment(void) * when we do a mostly-static build on OSX 10.7, the resulting binary won't * work on OSX 10.6. */ return *_NSGetEnviron(); -#else +#else /* !(defined(HAVE__NSGETENVIRON)) */ return environ; -#endif +#endif /* defined(HAVE__NSGETENVIRON) */ } /** Get name of current host and write it to <b>name</b> array, whose @@ -2549,6 +2580,7 @@ tor_inet_pton(int af, const char *src, void *dst) int gapPos = -1, i, setWords=0; const char *dot = strchr(src, '.'); const char *eow; /* end of words. */ + memset(words, 0xf8, sizeof(words)); if (dot == src) return 0; else if (!dot) @@ -2586,7 +2618,7 @@ tor_inet_pton(int af, const char *src, void *dst) long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd - * where treats "0xfoo" as an error, rather than as "0" followed by + * which treat "0xfoo" as an error, rather than as "0" followed by * "xfoo". */ return 0; } @@ -2673,7 +2705,8 @@ static int uname_result_is_set = 0; /** Return a pointer to a description of our platform. */ -MOCK_IMPL(const char *, get_uname, (void)) +MOCK_IMPL(const char *, +get_uname,(void)) { #ifdef HAVE_UNAME struct utsname u; @@ -2684,7 +2717,7 @@ MOCK_IMPL(const char *, get_uname, (void)) /* (Linux says 0 is success, Solaris says 1 is success) */ strlcpy(uname_result, u.sysname, sizeof(uname_result)); } else -#endif +#endif /* defined(HAVE_UNAME) */ { #ifdef _WIN32 OSVERSIONINFOEX info; @@ -2746,12 +2779,12 @@ MOCK_IMPL(const char *, get_uname, (void)) info.wProductType == VER_NT_DOMAIN_CONTROLLER) { strlcat(uname_result, " [server]", sizeof(uname_result)); } -#endif -#else +#endif /* defined(VER_NT_SERVER) */ +#else /* !(defined(_WIN32)) */ /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); /* LCOV_EXCL_STOP */ -#endif +#endif /* defined(_WIN32) */ } uname_result_is_set = 1; } @@ -2807,7 +2840,7 @@ compute_num_cpus_impl(void) return -1; #else return -1; -#endif +#endif /* defined(_WIN32) || ... */ } #define MAX_DETECTABLE_CPUS 16 @@ -2970,7 +3003,7 @@ tor_localtime_r(const time_t *timep, struct tm *result) memcpy(result, r, sizeof(struct tm)); return correct_tm(1, timep, result, r); } -#endif +#endif /* defined(HAVE_LOCALTIME_R) || ... */ /** @} */ /** @{ */ @@ -3013,9 +3046,13 @@ tor_gmtime_r(const time_t *timep, struct tm *result) memcpy(result, r, sizeof(struct tm)); return correct_tm(0, timep, result, r); } -#endif +#endif /* defined(HAVE_GMTIME_R) || ... */ #if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +#define HAVE_UNIX_MLOCKALL +#endif + +#ifdef HAVE_UNIX_MLOCKALL /** Attempt to raise the current and max rlimit to infinity for our process. * This only needs to be done once and can probably only be done when we have * not already dropped privileges. @@ -3046,7 +3083,7 @@ tor_set_max_memlock(void) return 0; } -#endif +#endif /* defined(HAVE_UNIX_MLOCKALL) */ /** Attempt to lock all current and all future memory pages. * This should only be called once and while we're privileged. @@ -3071,7 +3108,7 @@ tor_mlockall(void) * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx */ -#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +#ifdef HAVE_UNIX_MLOCKALL if (tor_set_max_memlock() == 0) { log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY."); } @@ -3092,10 +3129,10 @@ tor_mlockall(void) "pages: %s", strerror(errno)); return -1; } -#else +#else /* !(defined(HAVE_UNIX_MLOCKALL)) */ log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); return -1; -#endif +#endif /* defined(HAVE_UNIX_MLOCKALL) */ } /** @@ -3123,7 +3160,7 @@ tor_socket_errno(tor_socket_t sock) } return err; } -#endif +#endif /* defined(_WIN32) */ #if defined(_WIN32) #define E(code, s) { code, (s " [" #code " ]") } @@ -3199,7 +3236,7 @@ tor_socket_strerror(int e) } return strerror(e); } -#endif +#endif /* defined(_WIN32) */ /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -3225,7 +3262,7 @@ network_init(void) /* WSAData.iMaxSockets might show the max sockets we're allowed to use. * We might use it to complain if we're trying to be a server but have * too few sockets available. */ -#endif +#endif /* defined(_WIN32) */ return 0; } @@ -3246,7 +3283,7 @@ format_win32_error(DWORD err) FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), (LPVOID)&str, 0, NULL); @@ -3261,9 +3298,9 @@ format_win32_error(DWORD err) result = tor_malloc(len); wcstombs(result,str,len); result[len-1] = '\0'; -#else +#else /* !(defined(UNICODE)) */ result = tor_strdup(str); -#endif +#endif /* defined(UNICODE) */ } else { result = tor_strdup("<unformattable error>"); } @@ -3272,7 +3309,7 @@ format_win32_error(DWORD err) } return result; } -#endif +#endif /* defined(_WIN32) */ #if defined(HW_PHYSMEM64) /* This appears to be an OpenBSD thing */ @@ -3280,7 +3317,7 @@ format_win32_error(DWORD err) #elif defined(HW_MEMSIZE) /* OSX defines this one */ #define INT64_HW_MEM HW_MEMSIZE -#endif +#endif /* defined(HW_PHYSMEM64) || ... */ /** * Helper: try to detect the total system memory, and return it. On failure, @@ -3313,8 +3350,8 @@ get_total_system_memory_impl(void) tor_free(s); return result * 1024; - err: /* LCOV_EXCL_START Can't reach this unless proc is broken. */ + err: tor_free(s); close(fd); return 0; @@ -3354,7 +3391,7 @@ get_total_system_memory_impl(void) #else /* I have no clue. */ return 0; -#endif +#endif /* defined(__linux__) || ... */ } /** @@ -3387,7 +3424,7 @@ get_total_system_memory(size_t *mem_out) * size_t. */ m = SIZE_MAX; } -#endif +#endif /* SIZE_MAX != UINT64_MAX */ *mem_out = mem_cached = (size_t) m; @@ -3468,7 +3505,7 @@ tor_getpass(const char *prompt, char *output, size_t buflen) return r; #else #error "No implementation for tor_getpass found!" -#endif +#endif /* defined(HAVE_READPASSPHRASE) || ... */ } /** Return the amount of free disk space we have permission to use, in @@ -3508,6 +3545,6 @@ tor_get_avail_disk_space(const char *path) (void)path; errno = ENOSYS; return -1; -#endif +#endif /* defined(HAVE_STATVFS) || ... */ } diff --git a/src/common/compat.h b/src/common/compat.h index ee1c9454de..fee9e6587d 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_H @@ -50,8 +50,8 @@ * clang rejects because it is off the end of a less-than-3. Clang hates this, * even though those references never actually happen. */ # undef strcmp -# endif -#endif +#endif /* __has_feature(address_sanitizer) */ +#endif /* defined(__has_feature) */ #include <stdio.h> #include <errno.h> @@ -76,13 +76,13 @@ __attribute__ ((format(printf, formatIdx, firstArg))) #else #define CHECK_PRINTF(formatIdx, firstArg) -#endif +#endif /* defined(__GNUC__) */ #ifdef __GNUC__ #define CHECK_SCANF(formatIdx, firstArg) \ __attribute__ ((format(scanf, formatIdx, firstArg))) #else #define CHECK_SCANF(formatIdx, firstArg) -#endif +#endif /* defined(__GNUC__) */ /* What GCC do we have? */ #ifdef __GNUC__ @@ -109,18 +109,18 @@ PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(pop) -# else +#else /* !(defined(__clang__) || GCC_VERSION >= 406) */ /* older version of gcc: no push/pop support. */ # define DISABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) -# endif -#else /* ifdef __GNUC__ */ +#endif /* defined(__clang__) || GCC_VERSION >= 406 */ +#else /* !(defined(__GNUC__)) */ /* not gcc at all */ # define DISABLE_GCC_WARNING(warning) # define ENABLE_GCC_WARNING(warning) -#endif +#endif /* defined(__GNUC__) */ /* inline is __inline on windows. */ #ifdef _WIN32 @@ -142,9 +142,9 @@ #define __func__ __FUNC__ #else #define __func__ "???" -#endif -#endif /* ifndef MAVE_MACRO__func__ */ -#endif /* if not windows */ +#endif /* defined(HAVE_MACRO__FUNCTION__) || ... */ +#endif /* !defined(HAVE_MACRO__func__) */ +#endif /* defined(_MSC_VER) */ #define U64_TO_DBL(x) ((double) (x)) #define DBL_TO_U64(x) ((uint64_t) (x)) @@ -157,7 +157,7 @@ * problems), but if enumerated types are unsigned, we must use unsigned, * so that the loss of precision doesn't make large values negative. */ #define ENUM_BF(t) t -#endif +#endif /* defined(ENUM_VALS_ARE_SIGNED) */ /* GCC has several useful attributes. */ #if defined(__GNUC__) && __GNUC__ >= 3 @@ -194,7 +194,7 @@ * taken. This can generate slightly better code with some CPUs. */ #define PREDICT_UNLIKELY(exp) __builtin_expect(!!(exp), 0) -#else +#else /* !(defined(__GNUC__) && __GNUC__ >= 3) */ #define ATTR_NORETURN #define ATTR_CONST #define ATTR_MALLOC @@ -204,7 +204,7 @@ #define ATTR_WUR #define PREDICT_LIKELY(exp) (exp) #define PREDICT_UNLIKELY(exp) (exp) -#endif +#endif /* defined(__GNUC__) && __GNUC__ >= 3 */ /** Expands to a syntactically valid empty statement. */ #define STMT_NIL (void)0 @@ -224,7 +224,7 @@ #else #define STMT_BEGIN do { #define STMT_END } while (0) -#endif +#endif /* defined(__GNUC__) || ... */ /* Some tools (like coccinelle) don't like to see operators as macro * arguments. */ @@ -251,7 +251,7 @@ */ #undef strlcat #undef strlcpy -#endif +#endif /* defined __APPLE__ */ #ifndef HAVE_STRLCAT size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); @@ -272,24 +272,28 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); #define I64_PRINTF_ARG(a) (a) #define I64_SCANF_ARG(a) (a) #define I64_LITERAL(n) (n ## i64) -#else +#else /* !(defined(_MSC_VER)) */ #define U64_PRINTF_ARG(a) ((long long unsigned int)(a)) #define U64_SCANF_ARG(a) ((long long unsigned int*)(a)) #define U64_LITERAL(n) (n ## llu) #define I64_PRINTF_ARG(a) ((long long signed int)(a)) #define I64_SCANF_ARG(a) ((long long signed int*)(a)) #define I64_LITERAL(n) (n ## ll) +#endif /* defined(_MSC_VER) */ + +#if defined(__MINGW32__) || defined(__MINGW64__) +#define MINGW_ANY #endif -#if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) +#if defined(_MSC_VER) || defined(MINGW_ANY) /** The formatting string used to put a uint64_t value in a printf() or * scanf() function. See also U64_PRINTF_ARG and U64_SCANF_ARG. */ #define U64_FORMAT "%I64u" #define I64_FORMAT "%I64d" -#else +#else /* !(defined(_MSC_VER) || defined(MINGW_ANY)) */ #define U64_FORMAT "%llu" #define I64_FORMAT "%lld" -#endif +#endif /* defined(_MSC_VER) || defined(MINGW_ANY) */ #if (SIZEOF_INTPTR_T == SIZEOF_INT) #define INTPTR_T_FORMAT "%d" @@ -302,7 +306,7 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); #define INTPTR_PRINTF_ARG(x) I64_PRINTF_ARG(x) #else #error Unknown: SIZEOF_INTPTR_T -#endif +#endif /* (SIZEOF_INTPTR_T == SIZEOF_INT) || ... */ /** Represents an mmaped file. Allocated via tor_mmap_file; freed with * tor_munmap_file. */ @@ -316,7 +320,7 @@ typedef struct tor_mmap_t { * size, rounded up to the nearest page.) */ #elif defined _WIN32 HANDLE mmap_handle; -#endif +#endif /* defined(HAVE_SYS_MMAN_H) || ... */ } tor_mmap_t; @@ -378,7 +382,7 @@ const char *tor_fix_source_file(const char *fname); #else #define SHORT_FILE__ (__FILE__) #define tor_fix_source_file(s) (s) -#endif +#endif /* defined(_WIN32) */ /* ===== Time compatibility */ @@ -397,7 +401,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); (tvout)->tv_sec++; \ } \ } while (0) -#endif +#endif /* !defined(timeradd) */ #ifndef timersub /** Replacement for timersub on platforms that do not have it: sets tvout to @@ -411,13 +415,13 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); (tvout)->tv_sec--; \ } \ } while (0) -#endif +#endif /* !defined(timersub) */ #ifndef timercmp -/** Replacement for timersub on platforms that do not have it: returns true +/** Replacement for timercmp on platforms that do not have it: returns true * iff the relational operator "op" makes the expression tv1 op tv2 true. * - * Note that while this definition should work for all boolean opeators, some + * Note that while this definition should work for all boolean operators, some * platforms' native timercmp definitions do not support >=, <=, or ==. So * don't use those. */ @@ -425,7 +429,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); (((tv1)->tv_sec == (tv2)->tv_sec) ? \ ((tv1)->tv_usec op (tv2)->tv_usec) : \ ((tv1)->tv_sec op (tv2)->tv_sec)) -#endif +#endif /* !defined(timercmp) */ /* ===== File compatibility */ int tor_open_cloexec(const char *path, int flags, unsigned mode); @@ -467,7 +471,7 @@ typedef int socklen_t; #define TOR_SOCKET_T_FORMAT INTPTR_T_FORMAT #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET) #define TOR_INVALID_SOCKET INVALID_SOCKET -#else +#else /* !(defined(_WIN32)) */ /** Type used for a network socket. */ #define tor_socket_t int #define TOR_SOCKET_T_FORMAT "%d" @@ -475,7 +479,7 @@ typedef int socklen_t; #define SOCKET_OK(s) ((s) >= 0) /** Error/uninitialized value for a tor_socket_t. */ #define TOR_INVALID_SOCKET (-1) -#endif +#endif /* defined(_WIN32) */ int tor_close_socket_simple(tor_socket_t s); MOCK_DECL(int, tor_close_socket, (tor_socket_t s)); @@ -522,19 +526,19 @@ struct in6_addr #define s6_addr16 in6_u.u6_addr16 #define s6_addr32 in6_u.u6_addr32 }; -#endif +#endif /* !defined(HAVE_STRUCT_IN6_ADDR) */ /** @{ */ /** Many BSD variants seem not to define these. */ -#if defined(__APPLE__) || defined(__darwin__) || defined(__FreeBSD__) \ - || defined(__NetBSD__) || defined(__OpenBSD__) +#if defined(__APPLE__) || defined(__darwin__) || \ + defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) #ifndef s6_addr16 #define s6_addr16 __u6_addr.__u6_addr16 #endif #ifndef s6_addr32 #define s6_addr32 __u6_addr.__u6_addr32 #endif -#endif +#endif /* defined(__APPLE__) || defined(__darwin__) || ... */ /** @} */ #ifndef HAVE_SA_FAMILY_T @@ -566,7 +570,7 @@ struct sockaddr_in6 { struct in6_addr sin6_addr; // uint32_t sin6_scope_id; }; -#endif +#endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */ MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen)); int tor_inet_aton(const char *cp, struct in_addr *addr) ATTR_NONNULL((1,2)); @@ -607,14 +611,14 @@ int network_init(void); #define ERRNO_IS_EINTR(e) ((e) == WSAEINTR || 0) int tor_socket_errno(tor_socket_t sock); const char *tor_socket_strerror(int e); -#else +#else /* !(defined(_WIN32)) */ #define SOCK_ERRNO(e) e #if EAGAIN == EWOULDBLOCK /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ #define ERRNO_IS_EAGAIN(e) ((e) == EAGAIN || 0) #else #define ERRNO_IS_EAGAIN(e) ((e) == EAGAIN || (e) == EWOULDBLOCK) -#endif +#endif /* EAGAIN == EWOULDBLOCK */ #define ERRNO_IS_EINTR(e) ((e) == EINTR || 0) #define ERRNO_IS_EINPROGRESS(e) ((e) == EINPROGRESS || 0) #define ERRNO_IS_CONN_EINPROGRESS(e) ((e) == EINPROGRESS || 0) @@ -625,7 +629,7 @@ const char *tor_socket_strerror(int e); #define ERRNO_IS_EADDRINUSE(e) (((e) == EADDRINUSE) || 0) #define tor_socket_errno(sock) (errno) #define tor_socket_strerror(e) strerror(e) -#endif +#endif /* defined(_WIN32) */ /** Specified SOCKS5 status codes. */ typedef enum { @@ -728,7 +732,7 @@ char *format_win32_error(DWORD err); #define VER_SUITE_SINGLEUSERTS 0x00000100 #endif -#endif +#endif /* defined(_WIN32) */ #ifdef COMPAT_PRIVATE #if !defined(HAVE_SOCKETPAIR) || defined(_WIN32) || defined(TOR_UNIT_TESTS) @@ -736,12 +740,12 @@ char *format_win32_error(DWORD err); STATIC int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); #endif -#endif +#endif /* defined(COMPAT_PRIVATE) */ ssize_t tor_getpass(const char *prompt, char *output, size_t buflen); /* This needs some of the declarations above so we include it here. */ #include "compat_threads.h" -#endif +#endif /* !defined(TOR_COMPAT_H) */ diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 4a3b1af922..740cc2a11d 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -88,8 +88,8 @@ static struct event_base *the_event_base = NULL; (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) #else #define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif +#endif /* defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) */ +#endif /* defined(__APPLE__) */ /** Initialize the Libevent library and set up the event base. */ void @@ -237,8 +237,9 @@ tor_init_libevent_rng(void) return rv; } -#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) \ - && !defined(TOR_UNIT_TESTS) +#if defined(LIBEVENT_VERSION_NUMBER) && \ + LIBEVENT_VERSION_NUMBER >= V(2,1,1) && \ + !defined(TOR_UNIT_TESTS) void tor_gettimeofday_cached(struct timeval *tv) { @@ -249,7 +250,7 @@ tor_gettimeofday_cache_clear(void) { event_base_update_cache_time(the_event_base); } -#else +#else /* !(defined(LIBEVENT_VERSION_NUMBER) && ...) */ /** Cache the current hi-res time; the cache gets reset when libevent * calls us. */ static struct timeval cached_time_hires = {0, 0}; @@ -280,6 +281,15 @@ tor_gettimeofday_cache_set(const struct timeval *tv) tor_assert(tv); memcpy(&cached_time_hires, tv, sizeof(*tv)); } -#endif -#endif + +/** For testing: called post-fork to make libevent reinitialize + * kernel structures. */ +void +tor_libevent_postfork(void) +{ + int r = event_reinit(tor_libevent_get_base()); + tor_assert(r == 0); +} +#endif /* defined(TOR_UNIT_TESTS) */ +#endif /* defined(LIBEVENT_VERSION_NUMBER) && ... */ diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index c2e34764e4..834354c405 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_LIBEVENT_H @@ -54,6 +54,7 @@ void tor_gettimeofday_cached(struct timeval *tv); void tor_gettimeofday_cache_clear(void); #ifdef TOR_UNIT_TESTS void tor_gettimeofday_cache_set(const struct timeval *tv); +void tor_libevent_postfork(void); #endif #ifdef COMPAT_LIBEVENT_PRIVATE @@ -69,7 +70,7 @@ void tor_gettimeofday_cache_set(const struct timeval *tv); STATIC void libevent_logging_callback(int severity, const char *msg); -#endif +#endif /* defined(COMPAT_LIBEVENT_PRIVATE) */ -#endif +#endif /* !defined(TOR_COMPAT_LIBEVENT_H) */ diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h index 1bfe188075..c695f1e9df 100644 --- a/src/common/compat_openssl.h +++ b/src/common/compat_openssl.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_OPENSSL_H @@ -25,7 +25,7 @@ /* We define this macro if we're trying to build with the majorly refactored * API in OpenSSL 1.1 */ #define OPENSSL_1_1_API -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) && ... */ #ifndef OPENSSL_1_1_API #define OPENSSL_VERSION SSLEAY_VERSION @@ -37,11 +37,11 @@ ((st) == SSL3_ST_SW_SRVR_HELLO_B)) #define OSSL_HANDSHAKE_STATE int #define CONST_IF_OPENSSL_1_1_API -#else +#else /* !(!defined(OPENSSL_1_1_API)) */ #define STATE_IS_SW_SERVER_HELLO(st) \ ((st) == TLS_ST_SW_SRVR_HELLO) #define CONST_IF_OPENSSL_1_1_API const -#endif +#endif /* !defined(OPENSSL_1_1_API) */ -#endif +#endif /* !defined(TOR_COMPAT_OPENSSL_H) */ diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c index c1ae66c1d2..002274c469 100644 --- a/src/common/compat_pthreads.c +++ b/src/common/compat_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -201,20 +201,21 @@ tor_cond_init(tor_cond_t *cond) } #if defined(HAVE_CLOCK_GETTIME) -#if defined(CLOCK_MONOTONIC) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) +#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \ + defined(CLOCK_MONOTONIC) /* Use monotonic time so when we timedwait() on it, any clock adjustment * won't affect the timeout value. */ if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) { return -1; } #define USE_COND_CLOCK CLOCK_MONOTONIC -#else /* !defined HAVE_PTHREAD_CONDATTR_SETCLOCK */ +#else /* !(defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ...) */ /* On OSX Sierra, there is no pthread_condattr_setclock, so we are stuck * with the realtime clock. */ #define USE_COND_CLOCK CLOCK_REALTIME -#endif /* which clock to use */ -#endif /* HAVE_CLOCK_GETTIME */ +#endif /* defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ... */ +#endif /* defined(HAVE_CLOCK_GETTIME) */ if (pthread_cond_init(&cond->cond, &condattr)) { return -1; } @@ -266,11 +267,11 @@ tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, const struct timeval *tv) tvnow.tv_sec = ts.tv_sec; tvnow.tv_usec = (int)(ts.tv_nsec / 1000); timeradd(tv, &tvnow, &tvsum); -#else +#else /* !(defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK)) */ if (gettimeofday(&tvnow, NULL) < 0) return -1; timeradd(tv, &tvnow, &tvsum); -#endif /* HAVE_CLOCK_GETTIME, CLOCK_MONOTONIC */ +#endif /* defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK) */ ts.tv_sec = tvsum.tv_sec; ts.tv_nsec = tvsum.tv_usec * 1000; diff --git a/src/common/compat_rust.c b/src/common/compat_rust.c new file mode 100644 index 0000000000..366fd4037b --- /dev/null +++ b/src/common/compat_rust.c @@ -0,0 +1,39 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rust_compat.c + * \brief Rust FFI compatibility functions and helpers. This file is only built + * if Rust is not used. + **/ + +#include "compat_rust.h" +#include "util.h" + +/** + * Free storage pointed to by <b>str</b>, and itself. + */ +void +rust_str_free(rust_str_t str) +{ + char *s = (char *)str; + tor_free(s); +} + +/** + * Return zero-terminated contained string. + */ +const char * +rust_str_get(const rust_str_t str) +{ + return (const char *)str; +} + +/* If we were using Rust, we'd say so on startup. */ +rust_str_t +rust_welcome_string(void) +{ + char *s = tor_malloc_zero(1); + return (rust_str_t)s; +} + diff --git a/src/common/compat_rust.h b/src/common/compat_rust.h new file mode 100644 index 0000000000..72fde39296 --- /dev/null +++ b/src/common/compat_rust.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rust_compat.h + * \brief Headers for rust_compat.c + **/ + +#ifndef TOR_RUST_COMPAT_H +#define TOR_RUST_COMPAT_H + +#include "torint.h" + +/** + * Strings allocated in Rust must be freed from Rust code again. Let's make + * it less likely to accidentally mess up and call tor_free() on it, because + * currently it'll just work but might break at any time. + */ +typedef uintptr_t rust_str_t; + +void rust_str_free(rust_str_t); + +const char *rust_str_get(const rust_str_t); + +rust_str_t rust_welcome_string(void); + +#endif /* !defined(TOR_RUST_COMPAT_H) */ + diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index f4809060d6..208d3138d9 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -94,51 +94,73 @@ in_main_thread(void) } #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE) -/* As write(), but retry on EINTR */ +/* As write(), but retry on EINTR, and return the negative error code on + * error. */ static int write_ni(int fd, const void *buf, size_t n) { int r; again: r = (int) write(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } -/* As read(), but retry on EINTR */ +/* As read(), but retry on EINTR, and return the negative error code on error. + */ static int read_ni(int fd, void *buf, size_t n) { int r; again: r = (int) read(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } -#endif +#endif /* defined(HAVE_EVENTFD) || defined(HAVE_PIPE) */ -/** As send(), but retry on EINTR. */ +/** As send(), but retry on EINTR, and return the negative error code on + * error. */ static int send_ni(int fd, const void *buf, size_t n, int flags) { int r; again: r = (int) send(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } -/** As recv(), but retry on EINTR. */ +/** As recv(), but retry on EINTR, and return the negative error code on + * error. */ static int recv_ni(int fd, void *buf, size_t n, int flags) { int r; again: r = (int) recv(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } @@ -149,7 +171,7 @@ eventfd_alert(int fd) { uint64_t u = 1; int r = write_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) + if (r < 0 && -r != EAGAIN) return -1; return 0; } @@ -160,11 +182,11 @@ eventfd_drain(int fd) { uint64_t u = 0; int r = read_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return r; return 0; } -#endif +#endif /* defined(HAVE_EVENTFD) */ #ifdef HAVE_PIPE /** Send a byte over a pipe. Return 0 on success or EAGAIN; -1 on error */ @@ -172,8 +194,8 @@ static int pipe_alert(int fd) { ssize_t r = write_ni(fd, "x", 1); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return (int)r; return 0; } @@ -188,11 +210,11 @@ pipe_drain(int fd) r = read_ni(fd, buf, sizeof(buf)); } while (r > 0); if (r < 0 && errno != EAGAIN) - return -1; + return -errno; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } -#endif +#endif /* defined(HAVE_PIPE) */ /** Send a byte on socket <b>fd</b>t. Return 0 on success or EAGAIN, * -1 on error. */ @@ -200,13 +222,13 @@ static int sock_alert(tor_socket_t fd) { ssize_t r = send_ni(fd, "x", 1, 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; return 0; } /** Drain all the input from a socket <b>fd</b>, and ignore it. Return 0 on - * success, -1 on error. */ + * success, -errno on error. */ static int sock_drain(tor_socket_t fd) { @@ -215,8 +237,8 @@ sock_drain(tor_socket_t fd) do { r = recv_ni(fd, buf, sizeof(buf), 0); } while (r > 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } @@ -254,7 +276,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) socks_out->drain_fn = eventfd_drain; return 0; } -#endif +#endif /* defined(HAVE_EVENTFD) */ #ifdef HAVE_PIPE2 /* Now we're going to try pipes. First type the pipe2() syscall, if we @@ -267,7 +289,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) socks_out->drain_fn = pipe_drain; return 0; } -#endif +#endif /* defined(HAVE_PIPE2) */ #ifdef HAVE_PIPE /* Now try the regular pipe() syscall. Pipes have a bit lower overhead than @@ -291,7 +313,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) socks_out->drain_fn = pipe_drain; return 0; } -#endif +#endif /* defined(HAVE_PIPE) */ /* If nothing else worked, fall back on socketpair(). */ if (!(flags & ASOCKS_NOSOCKETPAIR) && @@ -330,3 +352,49 @@ alert_sockets_close(alert_sockets_t *socks) socks->read_fd = socks->write_fd = -1; } +/* + * XXXX We might be smart to move to compiler intrinsics or real atomic + * XXXX operations at some point. But not yet. + * + */ + +/** Initialize a new atomic counter with the value 0 */ +void +atomic_counter_init(atomic_counter_t *counter) +{ + memset(counter, 0, sizeof(*counter)); + tor_mutex_init_nonrecursive(&counter->mutex); +} +/** Clean up all resources held by an atomic counter. */ +void +atomic_counter_destroy(atomic_counter_t *counter) +{ + tor_mutex_uninit(&counter->mutex); + memset(counter, 0, sizeof(*counter)); +} +/** Add a value to an atomic counter. */ +void +atomic_counter_add(atomic_counter_t *counter, size_t add) +{ + tor_mutex_acquire(&counter->mutex); + counter->val += add; + tor_mutex_release(&counter->mutex); +} +/** Subtract a value from an atomic counter. */ +void +atomic_counter_sub(atomic_counter_t *counter, size_t sub) +{ + // this relies on unsigned overflow, but that's fine. + atomic_counter_add(counter, -sub); +} +/** Return the current value of an atomic counter */ +size_t +atomic_counter_get(atomic_counter_t *counter) +{ + size_t val; + tor_mutex_acquire(&counter->mutex); + val = counter->val; + tor_mutex_release(&counter->mutex); + return val; +} + diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h index 171a9f93ff..42f14eab2a 100644 --- a/src/common/compat_threads.h +++ b/src/common/compat_threads.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_THREADS_H @@ -20,7 +20,7 @@ #define USE_PTHREADS #else #error "No threading system was found" -#endif +#endif /* defined(_WIN32) || ... */ int spawn_func(void (*func)(void *), void *data); void spawn_exit(void) ATTR_NORETURN; @@ -41,7 +41,7 @@ typedef struct tor_mutex_t { #else /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */ int _unused; -#endif +#endif /* defined(USE_WIN32_THREADS) || ... */ } tor_mutex_t; tor_mutex_t *tor_mutex_new(void); @@ -73,7 +73,7 @@ typedef struct tor_cond_t { int generation; #else #error no known condition implementation. -#endif +#endif /* defined(USE_PTHREADS) || ... */ } tor_cond_t; tor_cond_t *tor_cond_new(void); @@ -147,5 +147,19 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal); */ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value); -#endif +/** + * Atomic counter type; holds a size_t value. + */ +typedef struct atomic_counter_t { + tor_mutex_t mutex; + size_t val; +} atomic_counter_t; + +void atomic_counter_init(atomic_counter_t *counter); +void atomic_counter_destroy(atomic_counter_t *counter); +void atomic_counter_add(atomic_counter_t *counter, size_t add); +void atomic_counter_sub(atomic_counter_t *counter, size_t sub); +size_t atomic_counter_get(atomic_counter_t *counter); + +#endif /* !defined(TOR_COMPAT_THREADS_H) */ diff --git a/src/common/compat_time.c b/src/common/compat_time.c index d044bbe1d7..1ce6f5ce4e 100644 --- a/src/common/compat_time.c +++ b/src/common/compat_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -28,7 +28,7 @@ /* as fallback implementation for tor_sleep_msec */ #include <sys/select.h> #endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ #ifdef __APPLE__ #include <mach/mach_time.h> @@ -64,9 +64,9 @@ tor_sleep_msec(int msec) select(0, NULL, NULL, NULL, &tv); #else sleep(CEIL_DIV(msec, 1000)); -#endif +#endif /* defined(_WIN32) || ... */ } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Set *timeval to the current time of day. On error, log and terminate. * (Same as gettimeofday(timeval,NULL), but never returns -1.) @@ -112,7 +112,7 @@ tor_gettimeofday(struct timeval *timeval) timeval->tv_usec = tb.millitm * 1000; #else #error "No way to get time." -#endif +#endif /* defined(_WIN32) || ... */ return; } @@ -187,8 +187,8 @@ monotime_coarse_set_mock_time_nsec(int64_t nsec) tor_assert_nonfatal(monotime_mocking_enabled == 1); mock_time_nsec_coarse = nsec; } -#endif -#endif +#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ +#endif /* defined(TOR_UNIT_TESTS) */ /* "ratchet" functions for monotonic time. */ @@ -235,7 +235,7 @@ ratchet_coarse_performance_counter(const int64_t count_raw) last_tick_count = count; return count; } -#endif +#endif /* defined(_WIN32) || defined(TOR_UNIT_TESTS) */ #if defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS) static struct timeval last_timeofday = { 0, 0 }; @@ -251,7 +251,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out) { /* must hold lock */ timeradd(timeval_raw, &timeofday_offset, out); - if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, <))) { + if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, OP_LT))) { /* time ran backwards. Instead, declare that no time occurred. */ timersub(&last_timeofday, timeval_raw, &timeofday_offset); memcpy(out, &last_timeofday, sizeof(struct timeval)); @@ -259,7 +259,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out) memcpy(&last_timeofday, out, sizeof(struct timeval)); } } -#endif +#endif /* defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS) */ #ifdef TOR_UNIT_TESTS /** For testing: reset all the ratchets */ @@ -271,7 +271,7 @@ monotime_reset_ratchets_for_testing(void) memset(&last_timeofday, 0, sizeof(struct timeval)); memset(&timeofday_offset, 0, sizeof(struct timeval)); } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ #ifdef __APPLE__ @@ -301,7 +301,7 @@ monotime_get(monotime_t *out) / mach_time_info.numer; return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ out->abstime_ = mach_absolute_time(); } @@ -332,7 +332,7 @@ monotime_diff_nsec(const monotime_t *start, * an old Linux kernel. In that case, we will fall back to CLOCK_MONOTONIC. */ static int clock_monotonic_coarse = CLOCK_MONOTONIC_COARSE; -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) */ static void monotime_init_internal(void) @@ -344,7 +344,7 @@ monotime_init_internal(void) "falling back to CLOCK_MONOTONIC.", strerror(errno)); clock_monotonic_coarse = CLOCK_MONOTONIC; } -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) */ } void @@ -356,7 +356,7 @@ monotime_get(monotime_t *out) out->ts_.tv_nsec = (int) (mock_time_nsec % ONE_BILLION); return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ int r = clock_gettime(CLOCK_MONOTONIC, &out->ts_); tor_assert(r == 0); } @@ -371,7 +371,7 @@ monotime_coarse_get(monotime_coarse_t *out) out->ts_.tv_nsec = (int) (mock_time_nsec_coarse % ONE_BILLION); return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ int r = clock_gettime(clock_monotonic_coarse, &out->ts_); if (PREDICT_UNLIKELY(r < 0) && errno == EINVAL && @@ -386,7 +386,7 @@ monotime_coarse_get(monotime_coarse_t *out) tor_assert(r == 0); } -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) */ int64_t monotime_diff_nsec(const monotime_t *start, @@ -462,7 +462,7 @@ monotime_get(monotime_t *out) / nsec_per_tick_numer; return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /* Alas, QueryPerformanceCounter is not always monotonic: see bug list at @@ -486,7 +486,7 @@ monotime_coarse_get(monotime_coarse_t *out) out->tick_count_ = mock_time_nsec_coarse / ONE_MILLION; return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ if (GetTickCount64_fn) { out->tick_count_ = (int64_t)GetTickCount64_fn(); @@ -570,7 +570,7 @@ monotime_diff_nsec(const monotime_t *start, /* end of "MONOTIME_USING_GETTIMEOFDAY" */ #else #error "No way to implement monotonic timers." -#endif +#endif /* defined(__APPLE__) || ... */ /** * Initialize the monotonic timer subsystem. Must be called before any @@ -653,5 +653,5 @@ monotime_coarse_absolute_msec(void) { return monotime_coarse_absolute_nsec() / ONE_MILLION; } -#endif +#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ diff --git a/src/common/compat_time.h b/src/common/compat_time.h index 2262446e57..5ea4aae42b 100644 --- a/src/common/compat_time.h +++ b/src/common/compat_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -28,13 +28,13 @@ #include <time.h> #endif -#if !defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) +#if !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) /** Implementation of timeval for platforms that don't have it. */ struct timeval { time_t tv_sec; unsigned int tv_usec; }; -#endif +#endif /* !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) */ /** Represents a monotonic timer in a platform-dependent way. */ typedef struct monotime_t { @@ -51,10 +51,11 @@ typedef struct monotime_t { #define MONOTIME_USING_GETTIMEOFDAY /* Otherwise, we will be stuck using gettimeofday. */ struct timeval tv_; -#endif +#endif /* defined(__APPLE__) || ... */ } monotime_t; -#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC_COARSE) +#if defined(CLOCK_MONOTONIC_COARSE) && \ + defined(HAVE_CLOCK_GETTIME) #define MONOTIME_COARSE_FN_IS_DIFFERENT #define monotime_coarse_t monotime_t #elif defined(_WIN32) @@ -66,7 +67,7 @@ typedef struct monotime_coarse_t { } monotime_coarse_t; #else #define monotime_coarse_t monotime_t -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) && ... || ... */ /** * Initialize the timing subsystem. This function is idempotent. @@ -109,12 +110,12 @@ void monotime_coarse_get(monotime_coarse_t *out); uint64_t monotime_coarse_absolute_nsec(void); uint64_t monotime_coarse_absolute_usec(void); uint64_t monotime_coarse_absolute_msec(void); -#else +#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ #define monotime_coarse_get monotime_get #define monotime_coarse_absolute_nsec monotime_absolute_nsec #define monotime_coarse_absolute_usec monotime_absolute_usec #define monotime_coarse_absolute_msec monotime_absolute_msec -#endif +#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start, @@ -123,11 +124,11 @@ int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start, const monotime_coarse_t *end); -#else +#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */ #define monotime_coarse_diff_nsec monotime_diff_nsec #define monotime_coarse_diff_usec monotime_diff_usec #define monotime_coarse_diff_msec monotime_diff_msec -#endif +#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ void tor_gettimeofday(struct timeval *timeval); @@ -142,7 +143,7 @@ void monotime_coarse_set_mock_time_nsec(int64_t); #else #define monotime_coarse_set_mock_time_nsec monotime_set_mock_time_nsec #endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ #ifdef COMPAT_TIME_PRIVATE #if defined(_WIN32) || defined(TOR_UNIT_TESTS) @@ -156,7 +157,7 @@ STATIC void ratchet_timeval(const struct timeval *timeval_raw, #ifdef TOR_UNIT_TESTS void monotime_reset_ratchets_for_testing(void); #endif -#endif +#endif /* defined(COMPAT_TIME_PRIVATE) */ -#endif +#endif /* !defined(TOR_COMPAT_TIME_H) */ diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c index 735be4ad17..50a3c498ca 100644 --- a/src/common/compat_winthreads.c +++ b/src/common/compat_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -246,5 +246,5 @@ tor_threads_init(void) set_main_thread(); } -#endif +#endif /* defined(_WIN32) */ diff --git a/src/common/compress.c b/src/common/compress.c new file mode 100644 index 0000000000..bc12a58ad6 --- /dev/null +++ b/src/common/compress.c @@ -0,0 +1,665 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.c + * \brief Common compression API. + **/ + +#include "orconfig.h" + +#include <stdlib.h> +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include "torint.h" + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" +#include "compress_none.h" +#include "compress_zlib.h" +#include "compress_zstd.h" + +/** Total number of bytes allocated for compression state overhead. */ +static atomic_counter_t total_compress_allocation; + +/** @{ */ +/* These macros define the maximum allowable compression factor. Anything of + * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to + * have an uncompression factor (uncompressed size:compressed size ratio) of + * any greater than MAX_UNCOMPRESSION_FACTOR. + * + * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to + * be small to limit the attack multiplier, but we also want it to be large + * enough so that no legitimate document --even ones we might invent in the + * future -- ever compresses by a factor of greater than + * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably + * large range of possible values. IMO, anything over 8 is probably safe; IMO + * anything under 50 is probably sufficient. + */ +#define MAX_UNCOMPRESSION_FACTOR 25 +#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ + +/** Return true if uncompressing an input of size <b>in_size</b> to an input of + * size at least <b>size_out</b> looks like a compression bomb. */ +MOCK_IMPL(int, +tor_compress_is_compression_bomb,(size_t size_in, size_t size_out)) +{ + if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) + return 0; + + return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); +} + +/** Guess the size that <b>in_len</b> will be after compression or + * decompression. */ +static size_t +guess_compress_size(int compress, compress_method_t method, + compression_level_t compression_level, + size_t in_len) +{ + // ignore these for now. + (void)compression_level; + if (method == NO_METHOD) { + /* Guess that we'll need an extra byte, to avoid a needless realloc + * for nul-termination */ + return (in_len < SIZE_MAX) ? in_len + 1 : in_len; + } + + /* Always guess a factor of 2. */ + if (compress) { + in_len /= 2; + } else { + if (in_len < SIZE_T_CEILING/2) + in_len *= 2; + } + return MAX(in_len, 1024); +} + +/** Internal function to implement tor_compress/tor_uncompress, depending on + * whether <b>compress</b> is set. All arguments are as for tor_compress or + * tor_uncompress. */ +static int +tor_compress_impl(int compress, + char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + compression_level_t compression_level, + int complete_only, + int protocol_warn_level) +{ + tor_compress_state_t *stream; + int rv; + + stream = tor_compress_new(compress, method, compression_level); + + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + return -1; + } + + size_t in_len_orig = in_len; + size_t out_remaining, out_alloc; + char *outptr; + + out_remaining = out_alloc = + guess_compress_size(compress, method, compression_level, in_len); + *out = outptr = tor_malloc(out_remaining); + + const int finish = complete_only || compress; + + while (1) { + switch (tor_compress_process(stream, + &outptr, &out_remaining, + &in, &in_len, finish)) { + case TOR_COMPRESS_DONE: + if (in_len == 0 || compress) { + goto done; + } else { + // More data is present, and we're decompressing. So we may need to + // reinitialize the stream if we are handling multiple concatenated + // inputs. + tor_compress_free(stream); + stream = tor_compress_new(compress, method, compression_level); + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + goto err; + } + } + break; + case TOR_COMPRESS_OK: + if (compress || complete_only) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "Unexpected %s while %scompressing", + complete_only?"end of input":"result", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + goto err; + } else { + if (in_len == 0) { + goto done; + } + } + break; + case TOR_COMPRESS_BUFFER_FULL: { + if (!compress && outptr < *out+out_alloc) { + // A buffer error in this case means that we have a problem + // with our input. + log_fn(protocol_warn_level, LD_PROTOCOL, + "Possible truncated or corrupt compressed data"); + goto err; + } + if (out_alloc >= SIZE_T_CEILING / 2) { + log_warn(LD_GENERAL, "While %scompressing data: ran out of space.", + compress?"":"un"); + goto err; + } + if (!compress && + tor_compress_is_compression_bomb(in_len_orig, out_alloc)) { + // This should already have been caught down in the backend logic. + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + const size_t offset = outptr - *out; + out_alloc *= 2; + *out = tor_realloc(*out, out_alloc); + outptr = *out + offset; + out_remaining = out_alloc - offset; + break; + } + case TOR_COMPRESS_ERROR: + log_fn(protocol_warn_level, LD_GENERAL, + "Error while %scompressing data: bad input?", + compress?"":"un"); + goto err; // bad data. + + // LCOV_EXCL_START + default: + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + } + done: + *out_len = outptr - *out; + if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a " + "compression bomb."); + goto err; + } + if (!compress) { + // NUL-terminate our output. + if (out_alloc == *out_len) + *out = tor_realloc(*out, out_alloc + 1); + (*out)[*out_len] = '\0'; + } + rv = 0; + goto out; + + err: + tor_free(*out); + *out_len = 0; + rv = -1; + goto out; + + out: + tor_compress_free(stream); + return rv; +} + +/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly + * allocated buffer, using the method described in <b>method</b>. Store the + * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. + * Return 0 on success, -1 on failure. + */ +int +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + return tor_compress_impl(1, out, out_len, in, in_len, method, + BEST_COMPRESSION, + 1, LOG_WARN); +} + +/** Given zero or more compressed strings of total length <b>in_len</b> bytes + * at <b>in</b>, uncompress them into a newly allocated buffer, using the + * method described in <b>method</b>. Store the uncompressed string in + * *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 on + * failure. + * + * If any bytes are written to <b>out</b>, an extra byte NUL is always + * written at the end, but not counted in <b>out_len</b>. This is a + * safety feature to ensure that the output can be treated as a + * NUL-terminated string -- though of course, callers should check + * out_len anyway. + * + * If <b>complete_only</b> is true, we consider a truncated input as a + * failure; otherwise we decompress as much as we can. Warn about truncated + * or corrupt inputs at <b>protocol_warn_level</b>. + */ +int +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + return tor_compress_impl(0, out, out_len, in, in_len, method, + BEST_COMPRESSION, + complete_only, protocol_warn_level); +} + +/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely + * to be compressed or not. If it is, return the likeliest compression method. + * Otherwise, return UNKNOWN_METHOD. + */ +compress_method_t +detect_compression_method(const char *in, size_t in_len) +{ + if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { + return GZIP_METHOD; + } else if (in_len > 2 && (in[0] & 0x0f) == 8 && + (ntohs(get_uint16(in)) % 31) == 0) { + return ZLIB_METHOD; + } else if (in_len > 2 && + fast_memeq(in, "\x5d\x00\x00", 3)) { + return LZMA_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) { + return ZSTD_METHOD; + } else { + return UNKNOWN_METHOD; + } +} + +/** Return 1 if a given <b>method</b> is supported; otherwise 0. */ +int +tor_compress_supports_method(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_method_supported(); + case LZMA_METHOD: + return tor_lzma_method_supported(); + case ZSTD_METHOD: + return tor_zstd_method_supported(); + case NO_METHOD: + return 1; + case UNKNOWN_METHOD: + default: + return 0; + } +} + +/** + * Return a bitmask of the supported compression types, where 1<<m is + * set in the bitmask if and only if compression with method <b>m</b> is + * supported. + */ +unsigned +tor_compress_get_supported_method_bitmask(void) +{ + static unsigned supported = 0; + if (supported == 0) { + compress_method_t m; + for (m = NO_METHOD; m <= UNKNOWN_METHOD; ++m) { + if (tor_compress_supports_method(m)) { + supported |= (1u << m); + } + } + } + return supported; +} + +/** Table of compression method names. These should have an "x-" prefix, + * if they are not listed in the IANA content coding registry. */ +static const struct { + const char *name; + compress_method_t method; +} compression_method_names[] = { + { "gzip", GZIP_METHOD }, + { "deflate", ZLIB_METHOD }, + // We call this "x-tor-lzma" rather than "x-lzma", because we impose a + // lower maximum memory usage on the decoding side. + { "x-tor-lzma", LZMA_METHOD }, + { "x-zstd" , ZSTD_METHOD }, + { "identity", NO_METHOD }, + + /* Later entries in this table are not canonical; these are recognized but + * not emitted. */ + { "x-gzip", GZIP_METHOD }, +}; + +/** Return the canonical string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (method == compression_method_names[i].method) + return compression_method_names[i].name; + } + return NULL; +} + +/** Table of compression human readable method names. */ +static const struct { + compress_method_t method; + const char *name; +} compression_method_human_names[] = { + { NO_METHOD, "uncompressed" }, + { GZIP_METHOD, "gzipped" }, + { ZLIB_METHOD, "deflated" }, + { LZMA_METHOD, "LZMA compressed" }, + { ZSTD_METHOD, "Zstandard compressed" }, + { UNKNOWN_METHOD, "unknown encoding" }, +}; + +/** Return a human readable string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_human_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) { + if (method == compression_method_human_names[i].method) + return compression_method_human_names[i].name; + } + return NULL; +} + +/** Return the compression method represented by the string <b>name</b>, or + * UNKNOWN_METHOD if the string isn't recognized. */ +compress_method_t +compression_method_get_by_name(const char *name) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (!strcmp(compression_method_names[i].name, name)) + return compression_method_names[i].method; + } + return UNKNOWN_METHOD; +} + +/** Return a string representation of the version of the library providing the + * compression method given in <b>method</b>. Returns NULL if <b>method</b> is + * unknown or unsupported. */ +const char * +tor_compress_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_version_str(); + case LZMA_METHOD: + return tor_lzma_get_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return a string representation of the version of the library, found at + * compile time, providing the compression method given in <b>method</b>. + * Returns NULL if <b>method</b> is unknown or unsupported. */ +const char * +tor_compress_header_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_header_version_str(); + case LZMA_METHOD: + return tor_lzma_get_header_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_header_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return the approximate number of bytes allocated for all + * supported compression schemas. */ +size_t +tor_compress_get_total_allocation(void) +{ + return atomic_counter_get(&total_compress_allocation) + + tor_zlib_get_total_allocation() + + tor_lzma_get_total_allocation() + + tor_zstd_get_total_allocation(); +} + +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { + compress_method_t method; /**< The compression method. */ + + union { + tor_zlib_compress_state_t *zlib_state; + tor_lzma_compress_state_t *lzma_state; + tor_zstd_compress_state_t *zstd_state; + } u; /**< Compression backend state. */ +}; + +/** Construct and return a tor_compress_state_t object using <b>method</b>. If + * <b>compress</b>, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress, compress_method_t method, + compression_level_t compression_level) +{ + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + break; + } + case LZMA_METHOD: { + tor_lzma_compress_state_t *lzma_state = + tor_lzma_compress_new(compress, method, compression_level); + + if (lzma_state == NULL) + goto err; + + state->u.lzma_state = lzma_state; + break; + } + case ZSTD_METHOD: { + tor_zstd_compress_state_t *zstd_state = + tor_zstd_compress_new(compress, method, compression_level); + + if (zstd_state == NULL) + goto err; + + state->u.zstd_state = zstd_state; + break; + } + case NO_METHOD: { + break; + } + case UNKNOWN_METHOD: + goto err; + } + + atomic_counter_add(&total_compress_allocation, + sizeof(tor_compress_state_t)); + return state; + + err: + tor_free(state); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + tor_assert(state != NULL); + const size_t in_len_orig = *in_len; + const size_t out_len_orig = *out_len; + tor_compress_output_t rv; + + if (*out_len == 0 && (*in_len > 0 || finish)) { + // If we still have input data, but no space for output data, we might as + // well return early and let the caller do the reallocation of the out + // variable. + return TOR_COMPRESS_BUFFER_FULL; + } + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + rv = tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + break; + case LZMA_METHOD: + rv = tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); + break; + case ZSTD_METHOD: + rv = tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); + break; + case NO_METHOD: + rv = tor_cnone_compress_process(out, out_len, in, in_len, + finish); + break; + default: + case UNKNOWN_METHOD: + goto err; + } + if (BUG((rv == TOR_COMPRESS_OK) && + *in_len == in_len_orig && + *out_len == out_len_orig)) { + log_warn(LD_GENERAL, + "More info on the bug: method == %s, finish == %d, " + " *in_len == in_len_orig == %lu, " + "*out_len == out_len_orig == %lu", + compression_method_get_human_name(state->method), finish, + (unsigned long)in_len_orig, (unsigned long)out_len_orig); + return TOR_COMPRESS_ERROR; + } + + return rv; + err: + return TOR_COMPRESS_ERROR; +} + +/** Deallocate <b>state</b>. */ +void +tor_compress_free(tor_compress_state_t *state) +{ + if (state == NULL) + return; + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + tor_zlib_compress_free(state->u.zlib_state); + break; + case LZMA_METHOD: + tor_lzma_compress_free(state->u.lzma_state); + break; + case ZSTD_METHOD: + tor_zstd_compress_free(state->u.zstd_state); + break; + case NO_METHOD: + break; + case UNKNOWN_METHOD: + break; + } + + atomic_counter_sub(&total_compress_allocation, + sizeof(tor_compress_state_t)); + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_compress_state_size(const tor_compress_state_t *state) +{ + tor_assert(state != NULL); + + size_t size = sizeof(tor_compress_state_t); + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + size += tor_zlib_compress_state_size(state->u.zlib_state); + break; + case LZMA_METHOD: + size += tor_lzma_compress_state_size(state->u.lzma_state); + break; + case ZSTD_METHOD: + size += tor_zstd_compress_state_size(state->u.zstd_state); + break; + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } + + return size; +} + +/** Initialize all compression modules. */ +void +tor_compress_init(void) +{ + atomic_counter_init(&total_compress_allocation); + + tor_zlib_init(); + tor_lzma_init(); + tor_zstd_init(); +} + diff --git a/src/common/compress.h b/src/common/compress.h new file mode 100644 index 0000000000..23a9817479 --- /dev/null +++ b/src/common/compress.h @@ -0,0 +1,90 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.h + * \brief Headers for compress.c + **/ + +#ifndef TOR_COMPRESS_H +#define TOR_COMPRESS_H + +/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and + * GZIP_METHOD is guaranteed to be supported by the compress/uncompress + * functions here. Call tor_compress_supports_method() to check if a given + * compression schema is supported by Tor. */ +typedef enum { + NO_METHOD=0, // This method must be first. + GZIP_METHOD=1, + ZLIB_METHOD=2, + LZMA_METHOD=3, + ZSTD_METHOD=4, + UNKNOWN_METHOD=5, // This method must be last. Add new ones in the middle. +} compress_method_t; + +/** + * Enumeration to define tradeoffs between memory usage and compression level. + * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most + * memory. + **/ +typedef enum { + BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION +} compression_level_t; + +int tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +compress_method_t detect_compression_method(const char *in, size_t in_len); + +MOCK_DECL(int,tor_compress_is_compression_bomb,(size_t size_in, + size_t size_out)); + +int tor_compress_supports_method(compress_method_t method); +unsigned tor_compress_get_supported_method_bitmask(void); +const char *compression_method_get_name(compress_method_t method); +const char *compression_method_get_human_name(compress_method_t method); +compress_method_t compression_method_get_by_name(const char *name); + +const char *tor_compress_version_str(compress_method_t method); + +const char *tor_compress_header_version_str(compress_method_t method); + +size_t tor_compress_get_total_allocation(void); + +/** Return values from tor_compress_process; see that function's documentation + * for details. */ +typedef enum { + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; + +/** Internal state for an incremental compression/decompression. */ +typedef struct tor_compress_state_t tor_compress_state_t; + +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); + +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free(tor_compress_state_t *state); + +size_t tor_compress_state_size(const tor_compress_state_t *state); + +void tor_compress_init(void); + +#endif /* !defined(TOR_COMPRESS_H) */ + diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c new file mode 100644 index 0000000000..6426ede4fd --- /dev/null +++ b/src/common/compress_lzma.c @@ -0,0 +1,361 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.c + * \brief Compression backend for LZMA. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" + +#ifdef HAVE_LZMA +#include <lzma.h> +#endif + +/** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */ +#define MEMORY_LIMIT (16 * 1024 * 1024) + +/** Total number of bytes allocated for LZMA state. */ +static atomic_counter_t total_lzma_allocation; + +#ifdef HAVE_LZMA +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 6; + case MEDIUM_COMPRESSION: return 4; + case LOW_COMPRESSION: return 2; + } +} + +/** Convert a given <b>error</b> to a human readable error string. */ +static const char * +lzma_error_str(lzma_ret error) +{ + switch (error) { + case LZMA_OK: + return "Operation completed successfully"; + case LZMA_STREAM_END: + return "End of stream"; + case LZMA_NO_CHECK: + return "Input stream lacks integrity check"; + case LZMA_UNSUPPORTED_CHECK: + return "Unable to calculate integrity check"; + case LZMA_GET_CHECK: + return "Integrity check available"; + case LZMA_MEM_ERROR: + return "Unable to allocate memory"; + case LZMA_MEMLIMIT_ERROR: + return "Memory limit reached"; + case LZMA_FORMAT_ERROR: + return "Unknown file format"; + case LZMA_OPTIONS_ERROR: + return "Unsupported options"; + case LZMA_DATA_ERROR: + return "Corrupt input data"; + case LZMA_BUF_ERROR: + return "Unable to progress"; + case LZMA_PROG_ERROR: + return "Programming error"; + default: + return "Unknown LZMA error"; + } +} +#endif /* defined(HAVE_LZMA) */ + +/** Return 1 if LZMA compression is supported; otherwise 0. */ +int +tor_lzma_method_supported(void) +{ +#ifdef HAVE_LZMA + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of liblzma. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_version_str(void) +{ +#ifdef HAVE_LZMA + return lzma_version_string(); +#else + return NULL; +#endif +} + +/** Return a string representation of the version of liblzma used at + * compilation time. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_header_version_str(void) +{ +#ifdef HAVE_LZMA + return LZMA_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal LZMA state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_lzma_compress_state_t { +#ifdef HAVE_LZMA + lzma_stream stream; /**< The LZMA stream. */ +#endif + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_LZMA +/** Return an approximate number of bytes stored in memory to hold the LZMA + * encoder/decoder state. */ +static size_t +tor_lzma_state_size_precalc(int compress, compression_level_t level) +{ + uint64_t memory_usage; + + if (compress) + memory_usage = lzma_easy_encoder_memusage(memory_level(level)); + else + memory_usage = lzma_easy_decoder_memusage(memory_level(level)); + + if (memory_usage == UINT64_MAX) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s", + compress ? "encoder" : "decoder"); + goto err; + // LCOV_EXCL_STOP + } + + if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX) + memory_usage = SIZE_MAX; + else + memory_usage += sizeof(tor_lzma_compress_state_t); + + return (size_t)memory_usage; + + // LCOV_EXCL_START + err: + return 0; + // LCOV_EXCL_STOP +} +#endif /* defined(HAVE_LZMA) */ + +/** Construct and return a tor_lzma_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == LZMA_METHOD); + +#ifdef HAVE_LZMA + tor_lzma_compress_state_t *result; + lzma_ret retval; + lzma_options_lzma stream_options; + + // Note that we do not explicitly initialize the lzma_stream object here, + // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is + // also what `tor_malloc_zero()` does. + result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); + result->compress = compress; + result->allocation = tor_lzma_state_size_precalc(compress, level); + + if (compress) { + lzma_lzma_preset(&stream_options, memory_level(level)); + + retval = lzma_alone_encoder(&result->stream, &stream_options); + + if (retval != LZMA_OK) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + // LCOV_EXCL_STOP + } + } else { + retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT); + + if (retval != LZMA_OK) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + // LCOV_EXCL_STOP + } + } + + atomic_counter_add(&total_lzma_allocation, result->allocation); + return result; + + /* LCOV_EXCL_START */ + err: + tor_free(result); + return NULL; + /* LCOV_EXCL_STOP */ +#else /* !(defined(HAVE_LZMA)) */ + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif /* defined(HAVE_LZMA) */ +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_LZMA + lzma_ret retval; + lzma_action action; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + state->stream.next_in = (unsigned char *)*in; + state->stream.avail_in = *in_len; + state->stream.next_out = (unsigned char *)*out; + state->stream.avail_out = *out_len; + + action = finish ? LZMA_FINISH : LZMA_RUN; + + retval = lzma_code(&state->stream, action); + + state->input_so_far += state->stream.next_in - ((unsigned char *)*in); + state->output_so_far += state->stream.next_out - ((unsigned char *)*out); + + *out = (char *)state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *)state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (retval) { + case LZMA_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_OK; + + case LZMA_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + + return TOR_COMPRESS_BUFFER_FULL; + + case LZMA_STREAM_END: + return TOR_COMPRESS_DONE; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + lzma_error_str(retval)); + return TOR_COMPRESS_ERROR; + } +#else /* !(defined(HAVE_LZMA)) */ + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + return TOR_COMPRESS_ERROR; +#endif /* defined(HAVE_LZMA) */ +} + +/** Deallocate <b>state</b>. */ +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_lzma_allocation, state->allocation); + +#ifdef HAVE_LZMA + lzma_end(&state->stream); +#endif + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all LZMA states. */ +size_t +tor_lzma_get_total_allocation(void) +{ + return atomic_counter_get(&total_lzma_allocation); +} + +/** Initialize the lzma module */ +void +tor_lzma_init(void) +{ + atomic_counter_init(&total_lzma_allocation); +} + diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h new file mode 100644 index 0000000000..7639d98a70 --- /dev/null +++ b/src/common/compress_lzma.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.h + * \brief Header for compress_lzma.c + **/ + +#ifndef TOR_COMPRESS_LZMA_H +#define TOR_COMPRESS_LZMA_H + +int tor_lzma_method_supported(void); + +const char *tor_lzma_get_version_str(void); + +const char *tor_lzma_get_header_version_str(void); + +/** Internal state for an incremental LZMA compression/decompression. */ +typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; + +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_lzma_compress_free(tor_lzma_compress_state_t *state); + +size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); + +size_t tor_lzma_get_total_allocation(void); + +void tor_lzma_init(void); + +#endif /* !defined(TOR_COMPRESS_LZMA_H) */ + diff --git a/src/common/compress_none.c b/src/common/compress_none.c new file mode 100644 index 0000000000..34314e4af7 --- /dev/null +++ b/src/common/compress_none.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.c + * \brief Compression backend for identity compression. + * + * We actually define this backend so that we can treat the identity transform + * as another case of compression. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_none.h" + +/** Transfer some bytes using the identity transformation. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + size_t n_to_copy = MIN(*in_len, *out_len); + + memcpy(*out, *in, n_to_copy); + *out += n_to_copy; + *in += n_to_copy; + *out_len -= n_to_copy; + *in_len -= n_to_copy; + if (*in_len == 0) { + return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK; + } else { + return TOR_COMPRESS_BUFFER_FULL; + } +} + diff --git a/src/common/compress_none.h b/src/common/compress_none.h new file mode 100644 index 0000000000..77c3cef47b --- /dev/null +++ b/src/common/compress_none.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.h + * \brief Header for compress_none.c + **/ + +#ifndef TOR_COMPRESS_NONE_H +#define TOR_COMPRESS_NONE_H + +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +#endif /* !defined(TOR_COMPRESS_NONE_H) */ + diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000000..284542e885 --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,304 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include <zlib.h> + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static atomic_counter_t total_zlib_allocation; + +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: return 9; + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + +/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return 1 if zlib/gzip compression is supported; otherwise 0. */ +int +tor_zlib_method_supported(void) +{ + /* We currently always support zlib/gzip, but we keep this function around in + * case we some day decide to deprecate zlib/gzip support. + */ + return 1; +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits <b>windowBits</b> and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress_, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress_) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = BEST_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress_; + bits = method_bits(method, compression_level); + memlevel = memory_level(compression_level); + if (compress_) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); + + atomic_counter_add(&total_zlib_allocation, out->allocation); + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + if (*in_len > UINT_MAX || + *out_len > UINT_MAX) { + return TOR_COMPRESS_ERROR; + } + + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : "<no message>"); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate <b>state</b>. */ +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zlib_allocation, state->allocation); + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return atomic_counter_get(&total_zlib_allocation); +} + +/** Set up global state for the zlib module */ +void +tor_zlib_init(void) +{ + atomic_counter_init(&total_zlib_allocation); +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000000..8ace467bf0 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +int tor_zlib_method_supported(void); + +const char *tor_zlib_get_version_str(void); + +const char *tor_zlib_get_header_version_str(void); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zlib_compress_free(tor_zlib_compress_state_t *state); + +size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t tor_zlib_get_total_allocation(void); + +void tor_zlib_init(void); + +#endif /* !defined(TOR_COMPRESS_ZLIB_H) */ + diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c new file mode 100644 index 0000000000..baa7749f0a --- /dev/null +++ b/src/common/compress_zstd.c @@ -0,0 +1,444 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.c + * \brief Compression backend for Zstandard. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zstd.h" + +#ifdef HAVE_ZSTD +DISABLE_GCC_WARNING(unused-const-variable) +#include <zstd.h> +ENABLE_GCC_WARNING(unused-const-variable) +#endif + +/** Total number of bytes allocated for Zstandard state. */ +static atomic_counter_t total_zstd_allocation; + +#ifdef HAVE_ZSTD +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 8; + case LOW_COMPRESSION: return 7; + } +} +#endif /* defined(HAVE_ZSTD) */ + +/** Return 1 if Zstandard compression is supported; otherwise 0. */ +int +tor_zstd_method_supported(void) +{ +#ifdef HAVE_ZSTD + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of libzstd. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_version_str(void) +{ +#ifdef HAVE_ZSTD + static char version_str[16]; + size_t version_number; + + version_number = ZSTD_versionNumber(); + tor_snprintf(version_str, sizeof(version_str), + "%d.%d.%d", + (int) version_number / 10000 % 100, + (int) version_number / 100 % 100, + (int) version_number % 100); + + return version_str; +#else /* !(defined(HAVE_ZSTD)) */ + return NULL; +#endif /* defined(HAVE_ZSTD) */ +} + +/** Return a string representation of the version of the version of libzstd + * used at compilation time. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_header_version_str(void) +{ +#ifdef HAVE_ZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal Zstandard state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zstd_compress_state_t { +#ifdef HAVE_ZSTD + union { + /** Compression stream. Used when <b>compress</b> is true. */ + ZSTD_CStream *compress_stream; + /** Decompression stream. Used when <b>compress</b> is false. */ + ZSTD_DStream *decompress_stream; + } u; /**< Zstandard stream objects. */ +#endif /* defined(HAVE_ZSTD) */ + + int compress; /**< True if we are compressing; false if we are inflating */ + int have_called_end; /**< True if we are compressing and we've called + * ZSTD_endStream */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_ZSTD +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ + tor_assert(preset > 0); + + size_t memory_usage = sizeof(tor_zstd_compress_state_t); + + // The Zstandard library provides a number of functions that would be useful + // here, but they are, unfortunately, still considered experimental and are + // thus only available in libzstd if we link against the library statically. + // + // The code in this function tries to approximate the calculations without + // being able to use the following: + // + // - We do not have access to neither the internal members of ZSTD_CStream + // and ZSTD_DStream and their internal context objects. + // + // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they + // are unexposed. + // + // In the future it might be useful to check if libzstd have started + // providing these functions in a stable manner and simplify this function. + if (compress) { + // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine: + memory_usage += 192; + + // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to + // variables that are not exposed via the public API. We use a _very_ + // simplified function to calculate the estimated amount of bytes used in + // this struct. + // memory_usage += (preset - 0.5) * 1024 * 1024; + memory_usage += (preset * 1024 * 1024) - (512 * 1024); + // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes. + // - stream->outBuffSize: 128 KB: + memory_usage += 128 * 1024; + // - stream->inBuffSize: 2048 KB: + memory_usage += 2048 * 1024; + } else { + // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine: + memory_usage += 208; + // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB. + memory_usage += 150 * 1024; + + // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes. + // - stream->inBuffSize: 0 KB. + // - stream->outBuffSize: 0 KB. + } + + return memory_usage; +} +#endif /* defined(HAVE_ZSTD) */ + +/** Construct and return a tor_zstd_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == ZSTD_METHOD); + +#ifdef HAVE_ZSTD + const int preset = memory_level(level); + tor_zstd_compress_state_t *result; + size_t retval; + + result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); + result->compress = compress; + result->allocation = tor_zstd_state_size_precalc(compress, preset); + + if (compress) { + result->u.compress_stream = ZSTD_createCStream(); + + if (result->u.compress_stream == NULL) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error while creating Zstandard compression " + "stream"); + goto err; + // LCOV_EXCL_STOP + } + + retval = ZSTD_initCStream(result->u.compress_stream, preset); + + if (ZSTD_isError(retval)) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + // LCOV_EXCL_STOP + } + } else { + result->u.decompress_stream = ZSTD_createDStream(); + + if (result->u.decompress_stream == NULL) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error while creating Zstandard decompression " + "stream"); + goto err; + // LCOV_EXCL_STOP + } + + retval = ZSTD_initDStream(result->u.decompress_stream); + + if (ZSTD_isError(retval)) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + // LCOV_EXCL_STOP + } + } + + atomic_counter_add(&total_zstd_allocation, result->allocation); + return result; + + err: + // LCOV_EXCL_START + if (compress) { + ZSTD_freeCStream(result->u.compress_stream); + } else { + ZSTD_freeDStream(result->u.decompress_stream); + } + + tor_free(result); + return NULL; + // LCOV_EXCL_STOP +#else /* !(defined(HAVE_ZSTD)) */ + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif /* defined(HAVE_ZSTD) */ +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_ZSTD + size_t retval; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + ZSTD_inBuffer input = { *in, *in_len, 0 }; + ZSTD_outBuffer output = { *out, *out_len, 0 }; + + if (BUG(finish == 0 && state->have_called_end)) { + finish = 1; + } + + if (state->compress) { + if (! state->have_called_end) + retval = ZSTD_compressStream(state->u.compress_stream, + &output, &input); + else + retval = 0; + } else { + retval = ZSTD_decompressStream(state->u.decompress_stream, + &output, &input); + } + + state->input_so_far += input.pos; + state->output_so_far += output.pos; + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + *in = (char *)input.src + input.pos; + *in_len = input.size - input.pos; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (state->compress && !state->have_called_end) { + retval = ZSTD_flushStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // ZSTD_flushStream returns 0 if the frame is done, or >0 if it + // is incomplete. + if (retval > 0) { + return TOR_COMPRESS_BUFFER_FULL; + } + } + + if (!finish) { + // The caller says we're not done with the input, so no need to write an + // epilogue. + return TOR_COMPRESS_OK; + } else if (state->compress) { + if (*in_len) { + // We say that we're not done with the input, so we can't write an + // epilogue. + return TOR_COMPRESS_OK; + } + + retval = ZSTD_endStream(state->u.compress_stream, &output); + state->have_called_end = 1; + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to write " + "epilogue: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // endStream returns the number of bytes that is needed to write the + // epilogue. + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_DONE; + } else /* if (!state->compress) */ { + // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it + // is incomplete. + // We check this above. + tor_assert_nonfatal(!ZSTD_isError(retval)); + // Start a new frame if this frame is done + if (retval == 0) + return TOR_COMPRESS_DONE; + // Don't check out_len, it might have some space left if the next output + // chunk is larger than the remaining space + else if (*in_len > 0) + return TOR_COMPRESS_BUFFER_FULL; + else + return TOR_COMPRESS_OK; + } + +#else /* !(defined(HAVE_ZSTD)) */ + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + + return TOR_COMPRESS_ERROR; +#endif /* defined(HAVE_ZSTD) */ +} + +/** Deallocate <b>state</b>. */ +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zstd_allocation, state->allocation); + +#ifdef HAVE_ZSTD + if (state->compress) { + ZSTD_freeCStream(state->u.compress_stream); + } else { + ZSTD_freeDStream(state->u.decompress_stream); + } +#endif /* defined(HAVE_ZSTD) */ + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all Zstandard + * states. */ +size_t +tor_zstd_get_total_allocation(void) +{ + return atomic_counter_get(&total_zstd_allocation); +} + +/** Initialize the zstd module */ +void +tor_zstd_init(void) +{ + atomic_counter_init(&total_zstd_allocation); +} + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h new file mode 100644 index 0000000000..02466010ff --- /dev/null +++ b/src/common/compress_zstd.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.h + * \brief Header for compress_zstd.c + **/ + +#ifndef TOR_COMPRESS_ZSTD_H +#define TOR_COMPRESS_ZSTD_H + +int tor_zstd_method_supported(void); + +const char *tor_zstd_get_version_str(void); + +const char *tor_zstd_get_header_version_str(void); + +/** Internal state for an incremental Zstandard compression/decompression. */ +typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; + +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zstd_compress_free(tor_zstd_compress_state_t *state); + +size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); + +size_t tor_zstd_get_total_allocation(void); + +void tor_zstd_init(void); + +#endif /* !defined(TOR_COMPRESS_ZSTD_H) */ + diff --git a/src/common/confline.c b/src/common/confline.c new file mode 100644 index 0000000000..04545bc2c3 --- /dev/null +++ b/src/common/confline.c @@ -0,0 +1,528 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "compat.h" +#include "confline.h" +#include "torlog.h" +#include "util.h" +#include "container.h" + +static int config_get_lines_aux(const char *string, config_line_t **result, + int extended, int allow_include, + int *has_include, int recursion_level, + config_line_t **last); +static smartlist_t *config_get_file_list(const char *path); +static int config_get_included_list(const char *path, int recursion_level, + int extended, config_line_t **list, + config_line_t **list_last); +static int config_process_include(const char *path, int recursion_level, + int extended, config_line_t **list, + config_line_t **list_last); + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * append it to *<b>lst</b>. */ +void +config_line_append(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = NULL; + while (*lst) + lst = &((*lst)->next); + + (*lst) = newline; +} + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * and prepend it to *<b>lst</b> */ +void +config_line_prepend(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = *lst; + *lst = newline; +} + +/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or + * NULL if no such key exists. + * + * (In options parsing, this is for handling commandline-only options only; + * other options should be looked up in the appropriate data structure.) */ +const config_line_t * +config_line_find(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcmp(cl->key, key)) + return cl; + } + return NULL; +} + +/** Auxiliary function that does all the work of config_get_lines. + * <b>recursion_level</b> is the count of how many nested %includes we have. + * Returns the a pointer to the last element of the <b>result</b> in + * <b>last</b>. */ +static int +config_get_lines_aux(const char *string, config_line_t **result, int extended, + int allow_include, int *has_include, int recursion_level, + config_line_t **last) +{ + config_line_t *list = NULL, **next, *list_last = NULL; + char *k, *v; + const char *parse_err; + int include_used = 0; + + if (recursion_level > MAX_INCLUDE_RECURSION_LEVEL) { + log_warn(LD_CONFIG, "Error while parsing configuration: more than %d " + "nested %%includes.", MAX_INCLUDE_RECURSION_LEVEL); + return -1; + } + + next = &list; + do { + k = v = NULL; + string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err); + if (!string) { + log_warn(LD_CONFIG, "Error while parsing configuration: %s", + parse_err?parse_err:"<unknown>"); + config_free_lines(list); + tor_free(k); + tor_free(v); + return -1; + } + if (k && v) { + unsigned command = CONFIG_LINE_NORMAL; + if (extended) { + if (k[0] == '+') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + command = CONFIG_LINE_APPEND; + } else if (k[0] == '/') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + tor_free(v); + v = tor_strdup(""); + command = CONFIG_LINE_CLEAR; + } + } + + if (allow_include && !strcmp(k, "%include")) { + tor_free(k); + include_used = 1; + + config_line_t *include_list; + if (config_process_include(v, recursion_level, extended, &include_list, + &list_last) < 0) { + log_warn(LD_CONFIG, "Error reading included configuration " + "file or directory: \"%s\".", v); + config_free_lines(list); + tor_free(v); + return -1; + } + *next = include_list; + if (list_last) + next = &list_last->next; + tor_free(v); + } else { + /* This list can get long, so we keep a pointer to the end of it + * rather than using config_line_append over and over and getting + * n^2 performance. */ + *next = tor_malloc_zero(sizeof(**next)); + (*next)->key = k; + (*next)->value = v; + (*next)->next = NULL; + (*next)->command = command; + list_last = *next; + next = &((*next)->next); + } + } else { + tor_free(k); + tor_free(v); + } + } while (*string); + + if (last) { + *last = list_last; + } + if (has_include) { + *has_include = include_used; + } + *result = list; + return 0; +} + +/** Helper: parse the config string and strdup into key/value + * strings. Set *result to the list, or NULL if parsing the string + * failed. Set *has_include to 1 if <b>result</b> has values from + * %included files. Return 0 on success, -1 on failure. Warn and ignore any + * misformatted lines. + * + * If <b>extended</b> is set, then treat keys beginning with / and with + as + * indicating "clear" and "append" respectively. */ +int +config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include) +{ + return config_get_lines_aux(string, result, extended, 1, has_include, 1, + NULL); +} + +/** Same as config_get_lines_include but does not allow %include */ +int +config_get_lines(const char *string, config_line_t **result, int extended) +{ + return config_get_lines_aux(string, result, extended, 0, NULL, 1, NULL); +} + +/** Adds a list of configuration files present on <b>path</b> to + * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file, + * only that file will be added to <b>file_list</b>. If it is a directory, + * all paths for files on that directory root (no recursion) except for files + * whose name starts with a dot will be added to <b>file_list</b>. + * Return 0 on success, -1 on failure. Ignores empty files. + */ +static smartlist_t * +config_get_file_list(const char *path) +{ + smartlist_t *file_list = smartlist_new(); + file_status_t file_type = file_status(path); + if (file_type == FN_FILE) { + smartlist_add_strdup(file_list, path); + return file_list; + } else if (file_type == FN_DIR) { + smartlist_t *all_files = tor_listdir(path); + if (!all_files) { + smartlist_free(file_list); + return NULL; + } + smartlist_sort_strings(all_files); + SMARTLIST_FOREACH_BEGIN(all_files, char *, f) { + if (f[0] == '.') { + tor_free(f); + continue; + } + + char *fullname; + tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f); + tor_free(f); + + if (file_status(fullname) != FN_FILE) { + tor_free(fullname); + continue; + } + smartlist_add(file_list, fullname); + } SMARTLIST_FOREACH_END(f); + smartlist_free(all_files); + return file_list; + } else if (file_type == FN_EMPTY) { + return file_list; + } else { + smartlist_free(file_list); + return NULL; + } +} + +/** Creates a list of config lines present on included <b>path</b>. + * Set <b>list</b> to the list and <b>list_last</b> to the last element of + * <b>list</b>. Return 0 on success, -1 on failure. */ +static int +config_get_included_list(const char *path, int recursion_level, int extended, + config_line_t **list, config_line_t **list_last) +{ + char *included_conf = read_file_to_str(path, 0, NULL); + if (!included_conf) { + return -1; + } + + if (config_get_lines_aux(included_conf, list, extended, 1, NULL, + recursion_level+1, list_last) < 0) { + tor_free(included_conf); + return -1; + } + + tor_free(included_conf); + return 0; +} + +/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the + * list of configuration settings obtained and <b>list_last</b> to the last + * element of the same list. Return 0 on success, -1 on failure. */ +static int +config_process_include(const char *path, int recursion_level, int extended, + config_line_t **list, config_line_t **list_last) +{ + config_line_t *ret_list = NULL; + config_line_t **next = &ret_list; +#if 0 + // Disabled -- we already unescape_string() on the result. */ + char *unquoted_path = get_unquoted_path(path); + if (!unquoted_path) { + return -1; + } + + smartlist_t *config_files = config_get_file_list(unquoted_path); + if (!config_files) { + tor_free(unquoted_path); + return -1; + } + tor_free(unquoted_path); +#endif /* 0 */ + smartlist_t *config_files = config_get_file_list(path); + if (!config_files) { + return -1; + } + + int rv = -1; + SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) { + config_line_t *included_list = NULL; + if (config_get_included_list(config_file, recursion_level, extended, + &included_list, list_last) < 0) { + goto done; + } + + *next = included_list; + if (*list_last) + next = &(*list_last)->next; + + } SMARTLIST_FOREACH_END(config_file); + *list = ret_list; + rv = 0; + + done: + SMARTLIST_FOREACH(config_files, char *, f, tor_free(f)); + smartlist_free(config_files); + return rv; +} + +/** + * Free all the configuration lines on the linked list <b>front</b>. + */ +void +config_free_lines(config_line_t *front) +{ + config_line_t *tmp; + + while (front) { + tmp = front; + front = tmp->next; + + tor_free(tmp->key); + tor_free(tmp->value); + tor_free(tmp); + } +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>. */ +config_line_t * +config_lines_dup(const config_line_t *inp) +{ + return config_lines_dup_and_filter(inp, NULL); +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>, + * but only the ones whose keys begin with <b>key</b> (case-insensitive). + * If <b>key</b> is NULL, do not filter. */ +config_line_t * +config_lines_dup_and_filter(const config_line_t *inp, + const char *key) +{ + config_line_t *result = NULL; + config_line_t **next_out = &result; + while (inp) { + if (key && strcasecmpstart(inp->key, key)) { + inp = inp->next; + continue; + } + *next_out = tor_malloc_zero(sizeof(config_line_t)); + (*next_out)->key = tor_strdup(inp->key); + (*next_out)->value = tor_strdup(inp->value); + inp = inp->next; + next_out = &((*next_out)->next); + } + (*next_out) = NULL; + return result; +} + +/** Return true iff a and b contain identical keys and values in identical + * order. */ +int +config_lines_eq(config_line_t *a, config_line_t *b) +{ + while (a && b) { + if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) + return 0; + a = a->next; + b = b->next; + } + if (a || b) + return 0; + return 1; +} + +/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */ +int +config_count_key(const config_line_t *a, const char *key) +{ + int n = 0; + while (a) { + if (!strcasecmp(a->key, key)) { + ++n; + } + a = a->next; + } + return n; +} + +/** Given a string containing part of a configuration file or similar format, + * advance past comments and whitespace and try to parse a single line. If we + * parse a line successfully, set *<b>key_out</b> to a new string holding the + * key portion and *<b>value_out</b> to a new string holding the value portion + * of the line, and return a pointer to the start of the next line. If we run + * out of data, return a pointer to the end of the string. If we encounter an + * error, return NULL and set *<b>err_out</b> (if provided) to an error + * message. + */ +const char * +parse_config_line_from_str_verbose(const char *line, char **key_out, + char **value_out, + const char **err_out) +{ + /* + See torrc_format.txt for a description of the (silly) format this parses. + */ + const char *key, *val, *cp; + int continuation = 0; + + tor_assert(key_out); + tor_assert(value_out); + + *key_out = *value_out = NULL; + key = val = NULL; + /* Skip until the first keyword. */ + while (1) { + while (TOR_ISSPACE(*line)) + ++line; + if (*line == '#') { + while (*line && *line != '\n') + ++line; + } else { + break; + } + } + + if (!*line) { /* End of string? */ + *key_out = *value_out = NULL; + return line; + } + + /* Skip until the next space or \ followed by newline. */ + key = line; + while (*line && !TOR_ISSPACE(*line) && *line != '#' && + ! (line[0] == '\\' && line[1] == '\n')) + ++line; + *key_out = tor_strndup(key, line-key); + + /* Skip until the value. */ + while (*line == ' ' || *line == '\t') + ++line; + + val = line; + + /* Find the end of the line. */ + if (*line == '\"') { // XXX No continuation handling is done here + if (!(line = unescape_string(line, value_out, NULL))) { + if (err_out) + *err_out = "Invalid escape sequence in quoted string"; + return NULL; + } + while (*line == ' ' || *line == '\t') + ++line; + if (*line == '\r' && *(++line) == '\n') + ++line; + if (*line && *line != '#' && *line != '\n') { + if (err_out) + *err_out = "Excess data after quoted string"; + return NULL; + } + } else { + /* Look for the end of the line. */ + while (*line && *line != '\n' && (*line != '#' || continuation)) { + if (*line == '\\' && line[1] == '\n') { + continuation = 1; + line += 2; + } else if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + if (*line == '\n') + ++line; + } else { + ++line; + } + } + + if (*line == '\n') { + cp = line++; + } else { + cp = line; + } + /* Now back cp up to be the last nonspace character */ + while (cp>val && TOR_ISSPACE(*(cp-1))) + --cp; + + tor_assert(cp >= val); + + /* Now copy out and decode the value. */ + *value_out = tor_strndup(val, cp-val); + if (continuation) { + char *v_out, *v_in; + v_out = v_in = *value_out; + while (*v_in) { + if (*v_in == '#') { + do { + ++v_in; + } while (*v_in && *v_in != '\n'); + if (*v_in == '\n') + ++v_in; + } else if (v_in[0] == '\\' && v_in[1] == '\n') { + v_in += 2; + } else { + *v_out++ = *v_in++; + } + } + *v_out = '\0'; + } + } + + if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + } + while (TOR_ISSPACE(*line)) ++line; + + return line; +} + diff --git a/src/common/confline.h b/src/common/confline.h new file mode 100644 index 0000000000..8256326f2d --- /dev/null +++ b/src/common/confline.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONFLINE_H +#define TOR_CONFLINE_H + +/** Ordinary configuration line. */ +#define CONFIG_LINE_NORMAL 0 +/** Appends to previous configuration for the same option, even if we + * would ordinary replace it. */ +#define CONFIG_LINE_APPEND 1 +/* Removes all previous configuration for an option. */ +#define CONFIG_LINE_CLEAR 2 + +#define MAX_INCLUDE_RECURSION_LEVEL 31 + +/** A linked list of lines in a config file, or elsewhere */ +typedef struct config_line_t { + char *key; + char *value; + struct config_line_t *next; + + /** What special treatment (if any) does this line require? */ + unsigned int command:2; + /** If true, subsequent assignments to this linelist should replace + * it, not extend it. Set only on the first item in a linelist in an + * or_options_t. */ + unsigned int fragile:1; +} config_line_t; + +void config_line_append(config_line_t **lst, + const char *key, const char *val); +void config_line_prepend(config_line_t **lst, + const char *key, const char *val); +config_line_t *config_lines_dup(const config_line_t *inp); +config_line_t *config_lines_dup_and_filter(const config_line_t *inp, + const char *key); +const config_line_t *config_line_find(const config_line_t *lines, + const char *key); +int config_lines_eq(config_line_t *a, config_line_t *b); +int config_count_key(const config_line_t *a, const char *key); +int config_get_lines(const char *string, config_line_t **result, int extended); +int config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include); +void config_free_lines(config_line_t *front); +const char *parse_config_line_from_str_verbose(const char *line, + char **key_out, char **value_out, + const char **err_out); +#endif /* !defined(TOR_CONFLINE_H) */ + diff --git a/src/common/container.c b/src/common/container.c index ec59dccf62..8645cb4826 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -132,6 +132,24 @@ smartlist_remove(smartlist_t *sl, const void *element) } } +/** As <b>smartlist_remove</b>, but do not change the order of + * any elements not removed */ +void +smartlist_remove_keeporder(smartlist_t *sl, const void *element) +{ + int i, j, num_used_orig = sl->num_used; + if (element == NULL) + return; + + for (i=j=0; j < num_used_orig; ++j) { + if (sl->list[j] == element) { + --sl->num_used; + } else { + sl->list[i++] = sl->list[j]; + } + } +} + /** If <b>sl</b> is nonempty, remove and return the final element. Otherwise, * return NULL. */ void * @@ -825,13 +843,13 @@ smartlist_sort_pointers(smartlist_t *sl) * } * * void timer_heap_insert(smartlist_t *heap, timer_t *timer) { - * smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index), + * smartlist_pqueue_add(heap, compare, offsetof(timer_t, heap_index), * timer); * } * * void timer_heap_pop(smartlist_t *heap) { * return smartlist_pqueue_pop(heap, compare, - * STRUCT_OFFSET(timer_t, heap_index)); + * offsetof(timer_t, heap_index)); * } */ diff --git a/src/common/container.h b/src/common/container.h index 71495b660a..f6affd3bc6 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONTAINER_H @@ -33,6 +33,7 @@ void smartlist_clear(smartlist_t *sl); void smartlist_add(smartlist_t *sl, void *element); void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2); void smartlist_remove(smartlist_t *sl, const void *element); +void smartlist_remove_keeporder(smartlist_t *sl, const void *element); void *smartlist_pop_last(smartlist_t *sl); void smartlist_reverse(smartlist_t *sl); void smartlist_string_remove(smartlist_t *sl, const char *element); @@ -73,11 +74,11 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) { tor_assert(sl->num_used > idx); sl->list[idx] = val; } -#else +#else /* !(defined(DEBUG_SMARTLIST)) */ #define smartlist_len(sl) ((sl)->num_used) #define smartlist_get(sl, idx) ((sl)->list[idx]) #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val)) -#endif +#endif /* defined(DEBUG_SMARTLIST) */ /** Exchange the elements at indices <b>idx1</b> and <b>idx2</b> of the * smartlist <b>sl</b>. */ @@ -223,6 +224,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ + (void) var ## _sl_idx; \ } STMT_END /** @@ -578,7 +580,7 @@ void* strmap_remove_lc(strmap_t *map, const char *key); #define BITARRAY_SHIFT 6 #else #error "int is neither 4 nor 8 bytes. I can't deal with that." -#endif +#endif /* SIZEOF_INT == 4 || ... */ #define BITARRAY_MASK ((1u<<BITARRAY_SHIFT)-1) /** A random-access array of one-bit-wide elements. */ @@ -721,5 +723,5 @@ third_quartile_uint32(uint32_t *array, int n_elements) return find_nth_uint32(array, n_elements, (n_elements*3)/4); } -#endif +#endif /* !defined(TOR_CONTAINER_H) */ diff --git a/src/common/crypto.c b/src/common/crypto.c index f8495bb107..16536f3716 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,7 +20,7 @@ /* Windows defines this; so does OpenSSL 0.9.8h and later. We don't actually * use either definition. */ #undef OCSP_RESPONSE -#endif +#endif /* defined(_WIN32) */ #define CRYPTO_PRIVATE #include "crypto.h" @@ -50,7 +50,7 @@ ENABLE_GCC_WARNING(redundant-decls) #else #pragma GCC diagnostic warning "-Wredundant-decls" #endif -#endif +#endif /* __GNUC__ && GCC_VERSION >= 402 */ #ifdef HAVE_CTYPE_H #include <ctype.h> @@ -101,7 +101,7 @@ ENABLE_GCC_WARNING(redundant-decls) * pointless, so let's not. */ #define NEW_THREAD_API -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && ... */ /** Longest recognized */ #define MAX_DNS_LABEL_SIZE 63 @@ -114,7 +114,7 @@ ENABLE_GCC_WARNING(redundant-decls) static tor_mutex_t **openssl_mutexes_ = NULL; /** How many mutexes have we allocated for use by OpenSSL? */ static int n_openssl_mutexes_ = 0; -#endif +#endif /* !defined(NEW_THREAD_API) */ /** A public key, or a public/private key-pair. */ struct crypto_pk_t @@ -198,7 +198,7 @@ log_engine(const char *fn, ENGINE *e) log_info(LD_CRYPTO, "Using default implementation for %s", fn); } } -#endif +#endif /* !defined(DISABLE_ENGINES) */ #ifndef DISABLE_ENGINES /** Try to load an engine in a shared library via fully qualified path. @@ -218,7 +218,7 @@ try_load_engine(const char *path, const char *engine) } return e; } -#endif +#endif /* !defined(DISABLE_ENGINES) */ /* Returns a trimmed and human-readable version of an openssl version string * <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10 @@ -394,7 +394,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) #else log_engine("ECDH", ENGINE_get_default_ECDH()); log_engine("ECDSA", ENGINE_get_default_ECDSA()); -#endif +#endif /* defined(OPENSSL_1_1_API) */ log_engine("RAND", ENGINE_get_default_RAND()); log_engine("RAND (which we will not use)", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); @@ -412,7 +412,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm)); #endif -#endif +#endif /* defined(DISABLE_ENGINES) */ } else { log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } @@ -450,9 +450,9 @@ crypto_pk_private_ok(const crypto_pk_t *k) const BIGNUM *p, *q; RSA_get0_factors(k->key, &p, &q); return p != NULL; /* XXX/yawning: Should we check q? */ -#else +#else /* !(defined(OPENSSL_1_1_API)) */ return k && k->key && k->key->p; -#endif +#endif /* defined(OPENSSL_1_1_API) */ } /** used by tortls.c: wrap an RSA* in a crypto_pk_t. */ @@ -467,7 +467,7 @@ crypto_new_pk_from_rsa_(RSA *rsa) return env; } -/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a +/** Helper, used by tor-gencert.c. Return the RSA from a * crypto_pk_t. */ RSA * crypto_pk_get_rsa_(crypto_pk_t *env) @@ -479,7 +479,7 @@ crypto_pk_get_rsa_(crypto_pk_t *env) * private is set, include the private-key portion of the key. Return a valid * pointer on success, and NULL on failure. */ MOCK_IMPL(EVP_PKEY *, - crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) +crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) { RSA *key = NULL; EVP_PKEY *pkey = NULL; @@ -516,7 +516,7 @@ crypto_dh_get_dh_(crypto_dh_t *dh) * be set. */ MOCK_IMPL(crypto_pk_t *, - crypto_pk_new,(void)) +crypto_pk_new,(void)) { RSA *rsa; @@ -606,7 +606,7 @@ crypto_cipher_free(crypto_cipher_t *env) * Return 0 on success, -1 on failure. */ MOCK_IMPL(int, - crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) +crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) { tor_assert(env); @@ -899,7 +899,7 @@ crypto_pk_public_exponent_ok(crypto_pk_t *env) RSA_get0_key(env->key, &n, &e, &d); #else e = env->key->e; -#endif +#endif /* defined(OPENSSL_1_1_API) */ return BN_is_word(e, 65537); } @@ -933,7 +933,7 @@ crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b) a_e = a->key->e; b_n = b->key->n; b_e = b->key->e; -#endif +#endif /* defined(OPENSSL_1_1_API) */ tor_assert(a_n != NULL && a_e != NULL); tor_assert(b_n != NULL && b_e != NULL); @@ -982,10 +982,10 @@ crypto_pk_num_bits(crypto_pk_t *env) tor_assert(n != NULL); return RSA_bits(env->key); -#else +#else /* !(defined(OPENSSL_1_1_API)) */ tor_assert(env->key->n); return BN_num_bits(env->key->n); -#endif +#endif /* defined(OPENSSL_1_1_API) */ } /** Increase the reference count of <b>env</b>, and return it. @@ -1012,7 +1012,7 @@ crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src) RSA_free(dest->key); dest->key = RSAPrivateKey_dup(src->key); } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Make a real honest-to-goodness copy of <b>env</b>, and return it. * Returns NULL on failure. */ @@ -1122,10 +1122,10 @@ crypto_pk_private_decrypt(crypto_pk_t *env, char *to, * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be * at least the length of the modulus of <b>env</b>. */ -int -crypto_pk_public_checksig(const crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen) +MOCK_IMPL(int, +crypto_pk_public_checksig,(const crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen)) { int r; tor_assert(env); @@ -1149,9 +1149,10 @@ crypto_pk_public_checksig(const crypto_pk_t *env, char *to, * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for * SHA1(data). Else return -1. */ -int -crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, size_t siglen) +MOCK_IMPL(int, +crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data, + size_t datalen, const char *sig, + size_t siglen)) { char digest[DIGEST_LEN]; char *buf; @@ -1252,9 +1253,12 @@ crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, * - The beginning of the source data prefixed with a 16-byte symmetric key, * padded and encrypted with the public key; followed by the rest of * the source data encrypted in AES-CTR mode with the symmetric key. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. */ int -crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, +crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen, @@ -1316,10 +1320,14 @@ crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, return -1; } -/** Invert crypto_pk_public_hybrid_encrypt. Returns the number of bytes - * written on success, -1 on failure. */ +/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of + * bytes written on success, -1 on failure. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. + */ int -crypto_pk_private_hybrid_decrypt(crypto_pk_t *env, +crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, @@ -1521,7 +1529,7 @@ crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out) if (crypto_pk_get_digest(pk, digest)) { return -1; } - if (crypto_digest(hashed_digest, digest, DIGEST_LEN)) { + if (crypto_digest(hashed_digest, digest, DIGEST_LEN) < 0) { return -1; } base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN); @@ -1715,19 +1723,21 @@ crypto_cipher_decrypt_with_iv(const char *key, /** Compute the SHA1 digest of the <b>len</b> bytes on data stored in * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. - * Return 0 on success, 1 on failure. + * Return 0 on success, -1 on failure. */ int crypto_digest(char *digest, const char *m, size_t len) { tor_assert(m); tor_assert(digest); - return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) + return -1; + return 0; } /** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result - * into <b>digest</b>. Return 0 on success, 1 on failure. */ + * into <b>digest</b>. Return 0 on success, -1 on failure. */ int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm) @@ -1735,16 +1745,22 @@ crypto_digest256(char *digest, const char *m, size_t len, tor_assert(m); tor_assert(digest); tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; if (algorithm == DIGEST_SHA256) - return (SHA256((const uint8_t*)m,len,(uint8_t*)digest) == NULL); + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); else - return (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - == -1); + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + + if (!ret) + return -1; + return 0; } /** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result - * into <b>digest</b>. Return 0 on success, 1 on failure. */ + * into <b>digest</b>. Return 0 on success, -1 on failure. */ int crypto_digest512(char *digest, const char *m, size_t len, digest_algorithm_t algorithm) @@ -1752,12 +1768,18 @@ crypto_digest512(char *digest, const char *m, size_t len, tor_assert(m); tor_assert(digest); tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; if (algorithm == DIGEST_SHA512) - return (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - == NULL); + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); else - return (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - == -1); + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + + if (!ret) + return -1; + return 0; } /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the @@ -1791,8 +1813,8 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg) return "sha3-256"; case DIGEST_SHA3_512: return "sha3-512"; - default: // LCOV_EXCL_START + default: tor_fragile_assert(); return "??unknown_digest??"; // LCOV_EXCL_STOP @@ -1854,6 +1876,18 @@ struct crypto_digest_t { } d; }; +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + /** * Return the number of bytes we need to malloc in order to get a * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe @@ -1865,7 +1899,7 @@ crypto_digest_alloc_bytes(digest_algorithm_t alg) /* Helper: returns the number of bytes in the 'f' field of 'st' */ #define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) /* Gives the length of crypto_digest_t through the end of the field 'd' */ -#define END_OF_FIELD(f) (STRUCT_OFFSET(crypto_digest_t, f) + \ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ STRUCT_FIELD_SIZE(crypto_digest_t, f)) switch (alg) { case DIGEST_SHA1: @@ -2124,6 +2158,35 @@ crypto_hmac_sha256(char *hmac_out, tor_assert(rv); } +/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a + * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length + * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in + * <b>mac_out</b>. This function can't fail. */ +void +crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len) +{ + crypto_digest_t *digest; + + const uint64_t key_len_netorder = tor_htonll(key_len); + + tor_assert(mac_out); + tor_assert(key); + tor_assert(msg); + + digest = crypto_digest256_new(DIGEST_SHA3_256); + + /* Order matters here that is any subsystem using this function should + * expect this very precise ordering in the MAC construction. */ + crypto_digest_add_bytes(digest, (const char *) &key_len_netorder, + sizeof(key_len_netorder)); + crypto_digest_add_bytes(digest, (const char *) key, key_len); + crypto_digest_add_bytes(digest, (const char *) msg, msg_len); + crypto_digest_get_digest(digest, (char *) mac_out, len_out); + crypto_digest_free(digest); +} + /** Internal state for a eXtendable-Output Function (XOF). */ struct crypto_xof_t { keccak_state s; @@ -2207,12 +2270,12 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) goto out; if (!DH_set0_pqg(dh, dh_p, NULL, dh_g)) goto out; -#else +#else /* !(defined(OPENSSL_1_1_API)) */ if (!(dh->p = BN_dup(p))) goto out; if (!(dh->g = BN_dup(g))) goto out; -#endif +#endif /* defined(OPENSSL_1_1_API) */ /* Perform the validation. */ int codes = 0; @@ -2383,7 +2446,7 @@ crypto_dh_new(int dh_type) if (!DH_set_length(res->dh, DH_PRIVATE_KEY_BITS)) goto err; -#else +#else /* !(defined(OPENSSL_1_1_API)) */ if (dh_type == DH_TYPE_TLS) { if (!(res->dh->p = BN_dup(dh_param_p_tls))) goto err; @@ -2396,12 +2459,13 @@ crypto_dh_new(int dh_type) goto err; res->dh->length = DH_PRIVATE_KEY_BITS; -#endif +#endif /* defined(OPENSSL_1_1_API) */ return res; - err: + /* LCOV_EXCL_START * This error condition is only reached when an allocation fails */ + err: crypto_log_errors(LOG_WARN, "creating DH object"); if (res->dh) DH_free(res->dh); /* frees p and g too */ tor_free(res); @@ -2458,7 +2522,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) "the-universe chances really do happen. Treating as a failure."); return -1; } -#else +#else /* !(defined(OPENSSL_1_1_API)) */ if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { /* LCOV_EXCL_START * If this happens, then openssl's DH implementation is busted. */ @@ -2471,7 +2535,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) goto again; /* LCOV_EXCL_STOP */ } -#endif +#endif /* defined(OPENSSL_1_1_API) */ return 0; } @@ -2492,7 +2556,7 @@ crypto_dh_get_public(crypto_dh_t *dh, char *pubkey, size_t pubkey_len) DH_get0_key(dh->dh, &dh_pub, &dh_priv); #else dh_pub = dh->dh->pub_key; -#endif +#endif /* defined(OPENSSL_1_1_API) */ if (!dh_pub) { if (crypto_dh_generate_public(dh)<0) @@ -2643,7 +2707,7 @@ crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len, for (cp = key_out, i=0; cp < key_out+key_out_len; ++i, cp += DIGEST_LEN) { tmp[key_in_len] = i; - if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1)) + if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1) < 0) goto exit; memcpy(cp, digest, MIN(DIGEST_LEN, key_out_len-(cp-key_out))); } @@ -2816,8 +2880,17 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) tor_assert(errno != EAGAIN); tor_assert(errno != EINTR); - /* Probably ENOSYS. */ - log_warn(LD_CRYPTO, "Can't get entropy from getrandom()."); + /* Useful log message for errno. */ + if (errno == ENOSYS) { + log_warn(LD_CRYPTO, "Can't get entropy from getrandom()." + " You are running a version of Tor built to support" + " getrandom(), but the kernel doesn't implement this" + " function--probably because it is too old?"); + } else { + log_warn(LD_CRYPTO, "Can't get entropy from getrandom(): %s.", + strerror(errno)); + } + getrandom_works = 0; /* Don't bother trying again. */ return -1; /* LCOV_EXCL_STOP */ @@ -2835,7 +2908,7 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) return getentropy(out, out_len); #else (void) out; -#endif +#endif /* defined(_WIN32) || ... */ /* This platform doesn't have a supported syscall based random. */ return -1; @@ -2859,7 +2932,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) (void)out; (void)out_len; return -1; -#else +#else /* !(defined(_WIN32)) */ static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; @@ -2867,7 +2940,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) size_t n; for (i = 0; filenames[i]; ++i) { - log_debug(LD_FS, "Opening %s for entropy", filenames[i]); + log_debug(LD_FS, "Considering %s for entropy", filenames[i]); fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); if (fd<0) continue; log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); @@ -2887,7 +2960,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) } return -1; -#endif +#endif /* defined(_WIN32) */ } /** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, @@ -3136,7 +3209,7 @@ crypto_rand_double(void) #define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 #else #error SIZEOF_INT is neither 4 nor 8 -#endif +#endif /* SIZEOF_INT == 4 || ... */ return ((double)u) / UINT_MAX_AS_DOUBLE; } @@ -3265,7 +3338,7 @@ memwipe(void *mem, uint8_t byte, size_t sz) **/ OPENSSL_cleanse(mem, sz); -#endif +#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */ /* Just in case some caller of memwipe() is relying on getting a buffer * filled with a particular value, fill the buffer. @@ -3307,7 +3380,7 @@ tor_set_openssl_thread_id(CRYPTO_THREADID *threadid) { CRYPTO_THREADID_set_numeric(threadid, tor_get_thread_id()); } -#endif +#endif /* !defined(NEW_THREAD_API) */ #if 0 /* This code is disabled, because OpenSSL never actually uses these callbacks. @@ -3357,7 +3430,7 @@ openssl_dynlock_destroy_cb_(struct CRYPTO_dynlock_value *v, tor_mutex_free(v->lock); tor_free(v); } -#endif +#endif /* 0 */ /** @{ */ /** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being @@ -3374,7 +3447,7 @@ setup_openssl_threading(void) openssl_mutexes_[i] = tor_mutex_new(); CRYPTO_set_locking_callback(openssl_locking_cb_); CRYPTO_THREADID_set_callback(tor_set_openssl_thread_id); -#endif +#endif /* !defined(NEW_THREAD_API) */ #if 0 CRYPTO_set_dynlock_create_callback(openssl_dynlock_create_cb_); CRYPTO_set_dynlock_lock_callback(openssl_dynlock_lock_cb_); @@ -3421,7 +3494,7 @@ crypto_global_cleanup(void) } tor_free(ms); } -#endif +#endif /* !defined(NEW_THREAD_API) */ tor_free(crypto_openssl_version_str); tor_free(crypto_openssl_header_version_str); @@ -3430,3 +3503,15 @@ crypto_global_cleanup(void) /** @} */ +#ifdef USE_DMALLOC +/** Tell the crypto library to use Tor's allocation functions rather than + * calling libc's allocation functions directly. Return 0 on success, -1 + * on failure. */ +int +crypto_use_tor_alloc_functions(void) +{ + int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_); + return r ? 0 : -1; +} +#endif /* defined(USE_DMALLOC) */ + diff --git a/src/common/crypto.h b/src/common/crypto.h index 116e0a62fd..f9aeeee2c0 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,6 +20,9 @@ #include "testsupport.h" #include "compat.h" +#include <openssl/engine.h> +#include "keccak-tiny/keccak-tiny.h" + /* Macro to create an arbitrary OpenSSL version number as used by OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard @@ -59,15 +62,20 @@ #define DIGEST256_LEN 32 /** Length of the output of our 64-bit optimized message digests (SHA512). */ #define DIGEST512_LEN 64 -/** Length of our symmetric cipher's keys. */ +/** Length of our symmetric cipher's keys of 128-bit. */ #define CIPHER_KEY_LEN 16 -/** Length of our symmetric cipher's IV. */ +/** Length of our symmetric cipher's IV of 128-bit. */ #define CIPHER_IV_LEN 16 +/** Length of our symmetric cipher's keys of 256-bit. */ +#define CIPHER256_KEY_LEN 32 /** Length of our public keys. */ #define PK_BYTES (1024/8) /** Length of our DH keys. */ #define DH_BYTES (1024/8) +/** Length of a sha1 message digest when encoded in base32 with trailing = + * signs removed. */ +#define BASE32_DIGEST_LEN 32 /** Length of a sha1 message digest when encoded in base64 with trailing = * signs removed. */ #define BASE64_DIGEST_LEN 27 @@ -129,6 +137,10 @@ int crypto_early_init(void) ATTR_WUR; int crypto_global_init(int hardwareAccel, const char *accelName, const char *accelPath) ATTR_WUR; +#ifdef USE_DMALLOC +int crypto_use_tor_alloc_functions(void); +#endif + void crypto_thread_cleanup(void); int crypto_global_cleanup(void); @@ -178,19 +190,21 @@ int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen, int padding, int warnOnFailure); -int crypto_pk_public_checksig(const crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen); -int crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, size_t siglen); +MOCK_DECL(int, crypto_pk_public_checksig,(const crypto_pk_t *env, + char *to, size_t tolen, + const char *from, size_t fromlen)); +MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env, + const char *data, size_t datalen, + const char *sig, size_t siglen)); int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen); int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen); -int crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, char *to, +int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen, int padding, int force); -int crypto_pk_private_hybrid_decrypt(crypto_pk_t *env, char *to, +int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen, int padding, int warnOnFailure); @@ -255,6 +269,10 @@ void crypto_digest_assign(crypto_digest_t *into, void crypto_hmac_sha256(char *hmac_out, const char *key, size_t key_len, const char *msg, size_t msg_len); +void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len); + crypto_xof_t *crypto_xof_new(void); void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); @@ -323,6 +341,7 @@ struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh); void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in); #ifdef CRYPTO_PRIVATE + STATIC int crypto_force_rand_ssleay(void); STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); @@ -330,11 +349,12 @@ STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); extern int break_strongest_rng_syscall; extern int break_strongest_rng_fallback; #endif -#endif +#endif /* defined(CRYPTO_PRIVATE) */ #ifdef TOR_UNIT_TESTS void crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src); +digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); #endif -#endif +#endif /* !defined(TOR_CRYPTO_H) */ diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index fcbee3aba2..8793fa6274 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -43,7 +43,7 @@ int curve25519_donna(uint8_t *mypublic, #elif defined(HAVE_NACL_CRYPTO_SCALARMULT_CURVE25519_H) #include <nacl/crypto_scalarmult_curve25519.h> #endif -#endif +#endif /* defined(USE_CURVE25519_NACL) */ static void pick_curve25519_basepoint_impl(void); @@ -72,7 +72,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret, r = crypto_scalarmult_curve25519(output, secret, bp); #else #error "No implementation of curve25519 is available." -#endif +#endif /* defined(USE_CURVE25519_DONNA) || ... */ memwipe(bp, 0, sizeof(bp)); return r; } @@ -80,7 +80,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret, /** * Helper function: Multiply the scalar "secret" by the Curve25519 * basepoint (X=9), and store the result in "output". Return 0 on - * success, -1 on false. + * success, -1 on failure. */ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret) @@ -318,8 +318,11 @@ curve25519_basepoint_spot_check(void) } goto end; + // LCOV_EXCL_START -- we can only hit this code if there is a bug in our + // curve25519-basepoint implementation. fail: r = -1; + // LCOV_EXCL_STOP end: curve25519_use_ed = save_use_ed; return r; diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 4011820949..d024ab79f5 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_CURVE25519_H @@ -71,7 +71,7 @@ STATIC int curve25519_impl(uint8_t *output, const uint8_t *secret, const uint8_t *basepoint); STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret); -#endif +#endif /* defined(CRYPTO_CURVE25519_PRIVATE) */ #define CURVE25519_BASE64_PADDED_LEN 44 @@ -83,5 +83,5 @@ int curve25519_public_to_base64(char *output, void curve25519_set_impl_params(int use_ed); void curve25519_init(void); -#endif +#endif /* !defined(TOR_CRYPTO_CURVE25519_H) */ diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index 30ed772274..94b23e31b9 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,6 +15,7 @@ * keys to and from the corresponding Curve25519 keys. */ +#define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> @@ -27,14 +28,12 @@ #include "crypto_format.h" #include "torlog.h" #include "util.h" +#include "util_format.h" #include "ed25519/ref10/ed25519_ref10.h" #include "ed25519/donna/ed25519_donna_tor.h" -#include <openssl/sha.h> - static void pick_ed25519_impl(void); -static int ed25519_impl_spot_check(void); /** An Ed25519 implementation, as a set of function pointers. */ typedef struct { @@ -59,6 +58,9 @@ typedef struct { int (*pubkey_from_curve25519_pubkey)(unsigned char *, const unsigned char *, int); + + int (*ed25519_scalarmult_with_group_order)(unsigned char *, + const unsigned char *); } ed25519_impl_t; /** The Ref10 Ed25519 implementation. This one is pure C and lightly @@ -79,6 +81,7 @@ static const ed25519_impl_t impl_ref10 = { ed25519_ref10_blind_public_key, ed25519_ref10_pubkey_from_curve25519_pubkey, + ed25519_ref10_scalarmult_with_group_order, }; /** The Ref10 Ed25519 implementation. This one is heavily optimized, but still @@ -99,6 +102,7 @@ static const ed25519_impl_t impl_donna = { ed25519_donna_blind_public_key, ed25519_donna_pubkey_from_curve25519_pubkey, + ed25519_donna_scalarmult_with_group_order, }; /** Which Ed25519 implementation are we using? NULL if we haven't decided @@ -147,7 +151,7 @@ crypto_ed25519_testing_restore_impl(void) ed25519_impl = saved_ed25519_impl; saved_ed25519_impl = NULL; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** * Initialize a new ed25519 secret key in <b>seckey_out</b>. If @@ -211,6 +215,14 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong) return 0; } +/** Return true iff 'pubkey' is set to zero (eg to indicate that it is not + * set). */ +int +ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey) +{ + return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); +} + /* Return a heap-allocated array that contains <b>msg</b> prefixed by the * string <b>prefix_str</b>. Set <b>final_msg_len_out</b> to the size of the * final array. If an error occured, return NULL. It's the resonsibility of the @@ -267,11 +279,11 @@ ed25519_sign(ed25519_signature_t *signature_out, * Like ed25519_sign(), but also prefix <b>msg</b> with <b>prefix_str</b> * before signing. <b>prefix_str</b> must be a NUL-terminated string. */ -int -ed25519_sign_prefixed(ed25519_signature_t *signature_out, - const uint8_t *msg, size_t msg_len, - const char *prefix_str, - const ed25519_keypair_t *keypair) +MOCK_IMPL(int, +ed25519_sign_prefixed,(ed25519_signature_t *signature_out, + const uint8_t *msg, size_t msg_len, + const char *prefix_str, + const ed25519_keypair_t *keypair)) { int retval; size_t prefixed_msg_len; @@ -281,9 +293,12 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out, prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str, &prefixed_msg_len); - if (!prefixed_msg) { + if (BUG(!prefixed_msg)) { + /* LCOV_EXCL_START -- only possible when the message and prefix are + * ridiculously huge */ log_warn(LD_GENERAL, "Failed to get prefixed msg."); return -1; + /* LCOV_EXCL_STOP */ } retval = ed25519_sign(signature_out, @@ -300,10 +315,10 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out, * * Return 0 if the signature is valid; -1 if it isn't. */ -int -ed25519_checksig(const ed25519_signature_t *signature, - const uint8_t *msg, size_t len, - const ed25519_public_key_t *pubkey) +MOCK_IMPL(int, +ed25519_checksig,(const ed25519_signature_t *signature, + const uint8_t *msg, size_t len, + const ed25519_public_key_t *pubkey)) { return get_ed_impl()->open(signature->sig, msg, len, pubkey->pubkey) < 0 ? -1 : 0; @@ -326,9 +341,12 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature, prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str, &prefixed_msg_len); - if (!prefixed_msg) { + if (BUG(!prefixed_msg)) { + /* LCOV_EXCL_START -- only possible when the message and prefix are + * ridiculously huge */ log_warn(LD_GENERAL, "Failed to get prefixed msg."); return -1; + /* LCOV_EXCL_STOP */ } retval = ed25519_checksig(signature, @@ -346,10 +364,10 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature, * was valid. Otherwise return -N, where N is the number of invalid * signatures. */ -int -ed25519_checksig_batch(int *okay_out, - const ed25519_checkable_t *checkable, - int n_checkable) +MOCK_IMPL(int, +ed25519_checksig_batch,(int *okay_out, + const ed25519_checkable_t *checkable, + int n_checkable)) { int i, res; const ed25519_impl_t *impl = get_ed_impl(); @@ -434,14 +452,16 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, { const char string[] = "Derive high part of ed25519 key from curve25519 key"; ed25519_public_key_t pubkey_check; - SHA512_CTX ctx; - uint8_t sha512_output[64]; + crypto_digest_t *ctx; + uint8_t sha512_output[DIGEST512_LEN]; memcpy(out->seckey.seckey, inp->seckey.secret_key, 32); - SHA512_Init(&ctx); - SHA512_Update(&ctx, out->seckey.seckey, 32); - SHA512_Update(&ctx, string, sizeof(string)); - SHA512_Final(sha512_output, &ctx); + + ctx = crypto_digest512_new(DIGEST_SHA512); + crypto_digest_add_bytes(ctx, (const char*)out->seckey.seckey, 32); + crypto_digest_add_bytes(ctx, (const char*)string, sizeof(string)); + crypto_digest_get_digest(ctx, (char *)sha512_output, sizeof(sha512_output)); + crypto_digest_free(ctx); memcpy(out->seckey.seckey + 32, sha512_output, 32); ed25519_public_key_generate(&out->pubkey, &out->seckey); @@ -454,7 +474,6 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32)); memwipe(&pubkey_check, 0, sizeof(pubkey_check)); - memwipe(&ctx, 0, sizeof(ctx)); memwipe(sha512_output, 0, sizeof(sha512_output)); return 0; @@ -483,7 +502,8 @@ ed25519_public_key_from_curve25519_public_key(ed25519_public_key_t *pubkey, * service descriptors are encrypted with a key derived from the service's * long-term public key, and then signed with (and stored at a position * indexed by) a short-term key derived by blinding the long-term keys. - */ + * + * Return 0 if blinding was successful, else return -1. */ int ed25519_keypair_blind(ed25519_keypair_t *out, const ed25519_keypair_t *inp, @@ -494,7 +514,9 @@ ed25519_keypair_blind(ed25519_keypair_t *out, get_ed_impl()->blind_secret_key(out->seckey.seckey, inp->seckey.seckey, param); - ed25519_public_blind(&pubkey_check, &inp->pubkey, param); + if (ed25519_public_blind(&pubkey_check, &inp->pubkey, param) < 0) { + return -1; + } ed25519_public_key_generate(&out->pubkey, &out->seckey); tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32)); @@ -514,8 +536,7 @@ ed25519_public_blind(ed25519_public_key_t *out, const ed25519_public_key_t *inp, const uint8_t *param) { - get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param); - return 0; + return get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param); } /** @@ -620,10 +641,22 @@ ed25519_pubkey_eq(const ed25519_public_key_t *key1, return tor_memeq(key1->pubkey, key2->pubkey, ED25519_PUBKEY_LEN); } +/** + * Set <b>dest</b> to contain the same key as <b>src</b>. + */ +void +ed25519_pubkey_copy(ed25519_public_key_t *dest, + const ed25519_public_key_t *src) +{ + tor_assert(dest); + tor_assert(src); + memcpy(dest, src, sizeof(ed25519_public_key_t)); +} + /** Check whether the given Ed25519 implementation seems to be working. * If so, return 0; otherwise return -1. */ -static int -ed25519_impl_spot_check(void) +MOCK_IMPL(STATIC int, +ed25519_impl_spot_check,(void)) { static const uint8_t alicesk[32] = { 0xc5,0xaa,0x8d,0xf4,0x3f,0x9f,0x83,0x7b, @@ -691,8 +724,11 @@ ed25519_impl_spot_check(void) */ goto end; + // LCOV_EXCL_START -- We can only reach this if our ed25519 implementation is + // broken. fail: r = -1; + // LCOV_EXCL_STOP end: return r; } @@ -734,3 +770,47 @@ ed25519_init(void) pick_ed25519_impl(); } +/* Return true if <b>point</b> is the identity element of the ed25519 group. */ +static int +ed25519_point_is_identity_element(const uint8_t *point) +{ + /* The identity element in ed25159 is the point with coordinates (0,1). */ + static const uint8_t ed25519_identity[32] = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + tor_assert(sizeof(ed25519_identity) == ED25519_PUBKEY_LEN); + return tor_memeq(point, ed25519_identity, sizeof(ed25519_identity)); +} + +/** Validate <b>pubkey</b> to ensure that it has no torsion component. + * Return 0 if <b>pubkey</b> is valid, else return -1. */ +int +ed25519_validate_pubkey(const ed25519_public_key_t *pubkey) +{ + uint8_t result[32] = {9}; + + /* First check that we were not given the identity element */ + if (ed25519_point_is_identity_element(pubkey->pubkey)) { + log_warn(LD_CRYPTO, "ed25519 pubkey is the identity"); + return -1; + } + + /* For any point on the curve, doing l*point should give the identity element + * (where l is the group order). Do the computation and check that the + * identity element is returned. */ + if (get_ed_impl()->ed25519_scalarmult_with_group_order(result, + pubkey->pubkey) < 0) { + log_warn(LD_CRYPTO, "ed25519 group order scalarmult failed"); + return -1; + } + + if (!ed25519_point_is_identity_element(result)) { + log_warn(LD_CRYPTO, "ed25519 validation failed"); + return -1; + } + + return 0; +} + diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h index 31afc49ccc..8d13a487d6 100644 --- a/src/common/crypto_ed25519.h +++ b/src/common/crypto_ed25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_ED25519_H @@ -51,21 +51,24 @@ int ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong); int ed25519_sign(ed25519_signature_t *signature_out, const uint8_t *msg, size_t len, const ed25519_keypair_t *key); -int ed25519_checksig(const ed25519_signature_t *signature, - const uint8_t *msg, size_t len, - const ed25519_public_key_t *pubkey); +MOCK_DECL(int,ed25519_checksig,(const ed25519_signature_t *signature, + const uint8_t *msg, size_t len, + const ed25519_public_key_t *pubkey)); + +MOCK_DECL(int, +ed25519_sign_prefixed,(ed25519_signature_t *signature_out, + const uint8_t *msg, size_t len, + const char *prefix_str, + const ed25519_keypair_t *keypair)); -int -ed25519_sign_prefixed(ed25519_signature_t *signature_out, - const uint8_t *msg, size_t len, - const char *prefix_str, - const ed25519_keypair_t *keypair); int ed25519_checksig_prefixed(const ed25519_signature_t *signature, const uint8_t *msg, size_t len, const char *prefix_str, const ed25519_public_key_t *pubkey); +int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey); + /** * A collection of information necessary to check an Ed25519 signature. Used * for batch verification. @@ -81,9 +84,9 @@ typedef struct { size_t len; } ed25519_checkable_t; -int ed25519_checksig_batch(int *okay_out, - const ed25519_checkable_t *checkable, - int n_checkable); +MOCK_DECL(int, ed25519_checksig_batch,(int *okay_out, + const ed25519_checkable_t *checkable, + int n_checkable)); int ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, int *signbit_out, @@ -118,14 +121,22 @@ void ed25519_keypair_free(ed25519_keypair_t *kp); int ed25519_pubkey_eq(const ed25519_public_key_t *key1, const ed25519_public_key_t *key2); +void ed25519_pubkey_copy(ed25519_public_key_t *dest, + const ed25519_public_key_t *src); void ed25519_set_impl_params(int use_donna); void ed25519_init(void); +int ed25519_validate_pubkey(const ed25519_public_key_t *pubkey); + #ifdef TOR_UNIT_TESTS void crypto_ed25519_testing_force_impl(const char *name); void crypto_ed25519_testing_restore_impl(void); #endif +#ifdef CRYPTO_ED25519_PRIVATE +MOCK_DECL(STATIC int, ed25519_impl_spot_check, (void)); #endif +#endif /* !defined(TOR_CRYPTO_ED25519_H) */ + diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index 2f6d847c83..1d090a8770 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -161,6 +161,27 @@ curve25519_public_from_base64(curve25519_public_key_t *pkey, } } +/** For logging convenience: Convert <b>pkey</b> to a statically allocated + * base64 string and return it. Not threadsafe. Format not meant to be + * computer-readable; it may change in the future. Subsequent calls invalidate + * previous returns. */ +const char * +ed25519_fmt(const ed25519_public_key_t *pkey) +{ + static char formatted[ED25519_BASE64_LEN+1]; + if (pkey) { + if (ed25519_public_key_is_zero(pkey)) { + strlcpy(formatted, "<unset>", sizeof(formatted)); + } else { + int r = ed25519_public_to_base64(formatted, pkey); + tor_assert(!r); + } + } else { + strlcpy(formatted, "<null>", sizeof(formatted)); + } + return formatted; +} + /** Try to decode the string <b>input</b> into an ed25519 public key. On * success, store the value in <b>pkey</b> and return 0. Otherwise return * -1. */ diff --git a/src/common/crypto_format.h b/src/common/crypto_format.h index 012e228cc4..bbd85dc720 100644 --- a/src/common/crypto_format.h +++ b/src/common/crypto_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_FORMAT_H @@ -28,6 +28,7 @@ int ed25519_public_from_base64(ed25519_public_key_t *pkey, const char *input); int ed25519_public_to_base64(char *output, const ed25519_public_key_t *pkey); +const char *ed25519_fmt(const ed25519_public_key_t *pkey); /* XXXX move these to crypto_format.h */ #define ED25519_SIG_BASE64_LEN 86 @@ -42,5 +43,5 @@ int digest_from_base64(char *digest, const char *d64); int digest256_to_base64(char *d64, const char *digest); int digest256_from_base64(char *digest, const char *d64); -#endif +#endif /* !defined(TOR_CRYPTO_FORMAT_H) */ diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 31e37c007d..12acc9331c 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -107,7 +107,6 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, rv = 0; goto out; - err: /* LCOV_EXCL_START This error case is often unreachable if we're correctly coded, unless @@ -123,6 +122,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, - pwbox_encoded_encode can't fail unless we're using trunnel wrong, or it's buggy. */ + err: tor_free(result); rv = -1; /* LCOV_EXCL_STOP */ diff --git a/src/common/crypto_pwbox.h b/src/common/crypto_pwbox.h index aadd477078..cee8653587 100644 --- a/src/common/crypto_pwbox.h +++ b/src/common/crypto_pwbox.h @@ -16,5 +16,5 @@ int crypto_unpwbox(uint8_t **out, size_t *outlen_out, const uint8_t *inp, size_t input_len, const char *secret, size_t secret_len); -#endif +#endif /* !defined(CRYPTO_PWBOX_H_INCLUDED_) */ diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index 5dbd2ad91f..b2fcca54c4 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -86,9 +86,11 @@ secret_to_key_key_len(uint8_t type) return DIGEST_LEN; case S2K_TYPE_SCRYPT: return DIGEST256_LEN; + // LCOV_EXCL_START default: - tor_fragile_assert(); // LCOV_EXCL_LINE - return -1; // LCOV_EXCL_LINE + tor_fragile_assert(); + return -1; + // LCOV_EXCL_STOP } } @@ -169,9 +171,11 @@ make_specifier(uint8_t *spec_out, uint8_t type, unsigned flags) /* r = 8; p = 2. */ spec_out[SCRYPT_SPEC_LEN-1] = (3u << 4) | (1u << 0); break; + // LCOV_EXCL_START - we should have returned above. default: - tor_fragile_assert(); // LCOV_EXCL_LINE - we should have returned above. + tor_fragile_assert(); return S2K_BAD_ALGORITHM; + // LCOV_EXCL_STOP } return speclen; @@ -290,9 +294,9 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv != 0) return S2K_FAILED; return (int)key_out_len; -#else +#else /* !(defined(HAVE_SCRYPT)) */ return S2K_NO_SCRYPT_SUPPORT; -#endif +#endif /* defined(HAVE_SCRYPT) */ } default: return S2K_BAD_ALGORITHM; diff --git a/src/common/crypto_s2k.h b/src/common/crypto_s2k.h index 9b186450b1..849ff59ce8 100644 --- a/src/common/crypto_s2k.h +++ b/src/common/crypto_s2k.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_S2K_H_INCLUDED @@ -67,7 +67,7 @@ STATIC int secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, const uint8_t *spec, size_t spec_len, const char *secret, size_t secret_len, int type); -#endif +#endif /* defined(CRYPTO_S2K_PRIVATE) */ -#endif +#endif /* !defined(TOR_CRYPTO_S2K_H_INCLUDED) */ diff --git a/src/common/di_ops.c b/src/common/di_ops.c index 4ed49e1164..7c0b4e7630 100644 --- a/src/common/di_ops.c +++ b/src/common/di_ops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -86,7 +86,7 @@ tor_memcmp(const void *a, const void *b, size_t len) } return retval; -#endif /* timingsafe_memcmp */ +#endif /* defined(HAVE_TIMINGSAFE_MEMCMP) */ } /** @@ -238,7 +238,7 @@ gt_i64_timei(uint64_t a, uint64_t b) int res = diff >> 63; return res & 1; } -#endif +#endif /* SIZEOF_VOID_P == 8 */ /** * Given an array of list of <b>n_entries</b> uint64_t values, whose sum is diff --git a/src/common/di_ops.h b/src/common/di_ops.h index 0a154302bf..e79973ba52 100644 --- a/src/common/di_ops.h +++ b/src/common/di_ops.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -46,5 +46,5 @@ int select_array_member_cumulative_timei(const uint64_t *entries, int n_entries, uint64_t total, uint64_t rand_val); -#endif +#endif /* !defined(TOR_DI_OPS_H) */ diff --git a/src/common/handles.h b/src/common/handles.h index 1ee2322579..a610753a1c 100644 --- a/src/common/handles.h +++ b/src/common/handles.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -149,5 +149,5 @@ } \ } -#endif /* TOR_HANDLE_H */ +#endif /* !defined(TOR_HANDLE_H) */ diff --git a/src/common/include.am b/src/common/include.am index cb307e9d5f..715ec0264c 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -82,9 +82,11 @@ LIBOR_A_SRC = \ src/common/address.c \ src/common/address_set.c \ src/common/backtrace.c \ + src/common/buffers.c \ src/common/compat.c \ src/common/compat_threads.c \ src/common/compat_time.c \ + src/common/confline.c \ src/common/container.c \ src/common/log.c \ src/common/memarea.c \ @@ -94,21 +96,32 @@ LIBOR_A_SRC = \ src/common/util_format.c \ src/common/util_process.c \ src/common/sandbox.c \ + src/common/storagedir.c \ src/common/workqueue.c \ $(libor_extra_source) \ $(threads_impl_source) \ $(readpassphrase_source) +if USE_RUST +else +LIBOR_A_SRC += src/common/compat_rust.c +endif + src/common/src_common_libor_testing_a-log.$(OBJEXT) \ src/common/log.$(OBJEXT): micro-revision.i LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/buffers_tls.c \ + src/common/compress.c \ + src/common/compress_lzma.c \ + src/common/compress_none.c \ + src/common/compress_zlib.c \ + src/common/compress_zstd.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ src/common/crypto_format.c \ - src/common/torgzip.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -138,13 +151,22 @@ COMMONHEADERS = \ src/common/address.h \ src/common/address_set.h \ src/common/backtrace.h \ + src/common/buffers.h \ + src/common/buffers_tls.h \ src/common/aes.h \ src/common/ciphers.inc \ src/common/compat.h \ src/common/compat_libevent.h \ src/common/compat_openssl.h \ + src/common/compat_rust.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress.h \ + src/common/compress_lzma.h \ + src/common/compress_none.h \ + src/common/compress_zlib.h \ + src/common/compress_zstd.h \ + src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ src/common/crypto_curve25519.h \ @@ -159,9 +181,9 @@ COMMONHEADERS = \ src/common/procmon.h \ src/common/pubsub.h \ src/common/sandbox.h \ + src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ - src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/log.c b/src/common/log.c index 4db1c9f0d0..e4d5cd8fd8 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -444,11 +444,11 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, if (m != msg_after_prefix) { tor_free(m); } -#else +#else /* !(defined(MAXLINE)) */ /* We have syslog but not MAXLINE. That's promising! */ syslog(severity, "%s", msg_after_prefix); -#endif -#endif +#endif /* defined(MAXLINE) */ +#endif /* defined(HAVE_SYSLOG_H) */ } else if (lf->callback) { if (domain & LD_NOCB) { if (!*callbacks_deferred && pending_cb_messages) { @@ -682,7 +682,7 @@ tor_log_get_logfile_names(smartlist_t *out) continue; if (lf->filename == NULL) continue; - smartlist_add(out, tor_strdup(lf->filename)); + smartlist_add_strdup(out, lf->filename); } UNLOCK_LOGS(); @@ -807,7 +807,7 @@ close_log(logfile_t *victim) /* There are no other syslogs; close the logging facility. */ closelog(); } -#endif +#endif /* defined(HAVE_SYSLOG_H) */ } } @@ -1086,7 +1086,7 @@ add_file_log(const log_severity_list_t *severity, const char *filename, int open_flags = O_WRONLY|O_CREAT; open_flags |= truncate_log ? O_TRUNC : O_APPEND; - fd = tor_open_cloexec(filename, open_flags, 0644); + fd = tor_open_cloexec(filename, open_flags, 0640); if (fd<0) return -1; if (tor_fd_seekend(fd)<0) { @@ -1144,7 +1144,7 @@ add_syslog_log(const log_severity_list_t *severity, UNLOCK_LOGS(); return 0; } -#endif +#endif /* defined(HAVE_SYSLOG_H) */ /** If <b>level</b> is a valid log severity, return the corresponding * numeric value. Otherwise, return -1. */ @@ -1177,7 +1177,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, @@ -1319,10 +1319,8 @@ parse_log_severity_config(const char **cfg_ptr, if (got_an_unqualified_range > 1) return -1; - space = strchr(cfg, ' '); + space = find_whitespace(cfg); dash = strchr(cfg, '-'); - if (!space) - space = strchr(cfg, '\0'); if (dash && dash < space) { sev_lo = tor_strndup(cfg, dash-cfg); sev_hi = tor_strndup(dash+1, space-(dash+1)); diff --git a/src/common/memarea.c b/src/common/memarea.c index 7d16b702e3..b059987e0e 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** \file memarea.c @@ -7,11 +7,15 @@ */ #include "orconfig.h" +#include <stddef.h> #include <stdlib.h> #include "memarea.h" #include "util.h" #include "compat.h" #include "torlog.h" +#include "container.h" + +#ifndef DISABLE_MEMORY_SENTINELS /** If true, we try to detect any attempts to write beyond the length of a * memarea. */ @@ -29,7 +33,7 @@ #define MEMAREA_ALIGN_MASK ((uintptr_t)7) #else #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." -#endif +#endif /* MEMAREA_ALIGN == 4 || ... */ #if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) #define USE_ALIGNED_ATTRIBUTE @@ -37,7 +41,7 @@ #define U_MEM mem #else #define U_MEM u.mem -#endif +#endif /* defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) */ #ifdef USE_SENTINELS /** Magic value that we stick at the end of a memarea so we can make sure @@ -57,11 +61,11 @@ uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END -#else +#else /* !(defined(USE_SENTINELS)) */ #define SENTINEL_LEN 0 #define SET_SENTINEL(chunk) STMT_NIL #define CHECK_SENTINEL(chunk) STMT_NIL -#endif +#endif /* defined(USE_SENTINELS) */ /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */ static inline void * @@ -93,12 +97,12 @@ typedef struct memarea_chunk_t { void *void_for_alignment_; /**< Dummy; used to make sure mem is aligned. */ } u; /**< Union used to enforce alignment when we don't have support for * doing it right. */ -#endif +#endif /* defined(USE_ALIGNED_ATTRIBUTE) */ } memarea_chunk_t; /** How many bytes are needed for overhead before we get to the memory part * of a chunk? */ -#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, U_MEM) +#define CHUNK_HEADER_SIZE offsetof(memarea_chunk_t, U_MEM) /** What's the smallest that we'll allocate a chunk? */ #define CHUNK_SIZE 4096 @@ -304,3 +308,91 @@ memarea_assert_ok(memarea_t *area) } } +#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */ + +struct memarea_t { + smartlist_t *pieces; +}; + +memarea_t * +memarea_new(void) +{ + memarea_t *ma = tor_malloc_zero(sizeof(memarea_t)); + ma->pieces = smartlist_new(); + return ma; +} +void +memarea_drop_all(memarea_t *area) +{ + memarea_clear(area); + smartlist_free(area->pieces); + tor_free(area); +} +void +memarea_clear(memarea_t *area) +{ + SMARTLIST_FOREACH(area->pieces, void *, p, tor_free_(p)); + smartlist_clear(area->pieces); +} +int +memarea_owns_ptr(const memarea_t *area, const void *ptr) +{ + SMARTLIST_FOREACH(area->pieces, const void *, p, if (ptr == p) return 1;); + return 0; +} + +void * +memarea_alloc(memarea_t *area, size_t sz) +{ + void *result = tor_malloc(sz); + smartlist_add(area->pieces, result); + return result; +} + +void * +memarea_alloc_zero(memarea_t *area, size_t sz) +{ + void *result = tor_malloc_zero(sz); + smartlist_add(area->pieces, result); + return result; +} +void * +memarea_memdup(memarea_t *area, const void *s, size_t n) +{ + void *r = memarea_alloc(area, n); + memcpy(r, s, n); + return r; +} +char * +memarea_strdup(memarea_t *area, const char *s) +{ + size_t n = strlen(s); + char *r = memarea_alloc(area, n+1); + memcpy(r, s, n); + r[n] = 0; + return r; +} +char * +memarea_strndup(memarea_t *area, const char *s, size_t n) +{ + size_t ln = strnlen(s, n); + char *r = memarea_alloc(area, ln+1); + memcpy(r, s, ln); + r[ln] = 0; + return r; +} +void +memarea_get_stats(memarea_t *area, + size_t *allocated_out, size_t *used_out) +{ + (void)area; + *allocated_out = *used_out = 128; +} +void +memarea_assert_ok(memarea_t *area) +{ + (void)area; +} + +#endif /* !defined(DISABLE_MEMORY_SENTINELS) */ + diff --git a/src/common/memarea.h b/src/common/memarea.h index 85bca51ad3..c3d954e1ce 100644 --- a/src/common/memarea.h +++ b/src/common/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Tor dependencies */ @@ -20,5 +20,5 @@ void memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out); void memarea_assert_ok(memarea_t *area); -#endif +#endif /* !defined(TOR_MEMAREA_H) */ diff --git a/src/common/procmon.c b/src/common/procmon.c index c485c760c7..26c11823e8 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -36,7 +36,7 @@ typedef int pid_t; #define PID_T_FORMAT I64_FORMAT #else #error Unknown: SIZEOF_PID_T -#endif +#endif /* (0 == SIZEOF_PID_T) && defined(_WIN32) || ... */ /* Define to 1 if process-termination monitors on this OS and Libevent version must poll for process termination themselves. */ @@ -71,7 +71,7 @@ parse_process_specifier(const char *process_spec, /* If we're lucky, long will turn out to be large enough to hold a * PID everywhere that Tor runs. */ - pid_l = tor_parse_long(process_spec, 0, 1, LONG_MAX, &pid_ok, &pspec_next); + pid_l = tor_parse_long(process_spec, 10, 1, LONG_MAX, &pid_ok, &pspec_next); /* Reserve room in the ‘process specifier’ for additional * (platform-specific) identifying information beyond the PID, to @@ -114,7 +114,7 @@ struct tor_process_monitor_t { HANDLE hproc; /* XXXX We should have Libevent watch hproc for us, * if/when some version of Libevent can be told to do so. */ -#endif +#endif /* defined(_WIN32) */ /* XXXX On Linux, we can and should receive the 22nd * (space-delimited) field (‘starttime’) of /proc/$PID/stat from the @@ -219,7 +219,7 @@ tor_process_monitor_new(struct event_base *base, "try again later.", procmon->pid); } -#endif +#endif /* defined(_WIN32) */ procmon->cb = cb; procmon->cb_arg = cb_arg; @@ -232,9 +232,9 @@ tor_process_monitor_new(struct event_base *base, * tor_evtimer_new never returns NULL. */ evtimer_add(procmon->e, &poll_interval_tv); -#else +#else /* !(defined(PROCMON_POLLS)) */ #error OOPS? -#endif +#endif /* defined(PROCMON_POLLS) */ return procmon; err: @@ -306,11 +306,11 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, tor_free(errmsg); } } -#else +#else /* !(defined(_WIN32)) */ /* Unix makes this part easy, if a bit racy. */ its_dead_jim = kill(procmon->pid, 0); its_dead_jim = its_dead_jim && (errno == ESRCH); -#endif +#endif /* defined(_WIN32) */ tor_log(its_dead_jim ? LOG_NOTICE : LOG_INFO, procmon->log_domain, "Monitored process "PID_T_FORMAT" is %s.", @@ -321,7 +321,7 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, procmon->cb(procmon->cb_arg); } } -#endif +#endif /* defined(PROCMON_POLLS) */ /** Free the process-termination monitor <b>procmon</b>. */ void diff --git a/src/common/procmon.h b/src/common/procmon.h index 49ead24092..10ead11ba8 100644 --- a/src/common/procmon.h +++ b/src/common/procmon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,5 +29,5 @@ tor_process_monitor_t *tor_process_monitor_new(struct event_base *base, const char **msg); void tor_process_monitor_free(tor_process_monitor_t *procmon); -#endif +#endif /* !defined(TOR_PROCMON_H) */ diff --git a/src/common/pubsub.c b/src/common/pubsub.c index b3faf40e00..336e8a6e7f 100644 --- a/src/common/pubsub.c +++ b/src/common/pubsub.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.h b/src/common/pubsub.h index bbb4f02a42..2bee3af085 100644 --- a/src/common/pubsub.h +++ b/src/common/pubsub.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -175,5 +175,5 @@ int pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn, pubsub_clear_(&name##_topic_); \ } -#endif /* TOR_PUBSUB_H */ +#endif /* !defined(TOR_PUBSUB_H) */ diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 3d27ea66b5..97acf894f3 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,10 +17,16 @@ * with the libevent fix. */ #define _LARGEFILE64_SOURCE -#endif +#endif /* !defined(_LARGEFILE64_SOURCE) */ -/** Malloc mprotect limit in bytes. */ -#define MALLOC_MP_LIM 1048576 +/** Malloc mprotect limit in bytes. + * + * 28/06/2017: This value was increased from 16 MB to 20 MB after we introduced + * LZMA support in Tor (0.3.1.1-alpha). We limit our LZMA coder to 16 MB, but + * liblzma have a small overhead that we need to compensate for to avoid being + * killed by the sandbox. + */ +#define MALLOC_MP_LIM (20*1024*1024) #include <stdio.h> #include <string.h> @@ -74,7 +80,7 @@ #define USE_BACKTRACE #define EXPOSE_CLEAN_BACKTRACE #include "backtrace.h" -#endif +#endif /* defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && ... */ #ifdef USE_BACKTRACE #include <execinfo.h> @@ -100,7 +106,12 @@ #define M_SYSCALL arm_r7 -#endif +#elif defined(__aarch64__) && defined(__LP64__) + +#define REG_SYSCALL 8 +#define M_SYSCALL regs[REG_SYSCALL] + +#endif /* defined(__i386__) || ... */ /**Determines if at least one sandbox is active.*/ static int sandbox_active = 0; @@ -293,37 +304,6 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return rc; } -#if 0 -/** - * Function responsible for setting up the execve syscall for - * the seccomp filter sandbox. - */ -static int -sb_execve(scmp_filter_ctx ctx, sandbox_cfg_t *filter) -{ - int rc; - sandbox_cfg_t *elem = NULL; - - // for each dynamic parameter filters - for (elem = filter; elem != NULL; elem = elem->next) { - smp_param_t *param = elem->param; - - if (param != NULL && param->prot == 1 && param->syscall - == SCMP_SYS(execve)) { - rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(execve), - SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); - if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add execve syscall, received " - "libseccomp error %d", rc); - return rc; - } - } - } - - return 0; -} -#endif - /** * Function responsible for setting up the time syscall for * the seccomp filter sandbox. @@ -337,7 +317,7 @@ sb_time(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(0, SCMP_CMP_EQ, 0)); #else return 0; -#endif +#endif /* defined(__NR_time) */ } /** @@ -356,7 +336,7 @@ sb_accept4(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (rc) { return rc; } -#endif +#endif /* defined(__i386__) */ rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(accept4), SCMP_CMP_MASKED(3, SOCK_CLOEXEC|SOCK_NONBLOCK, 0)); @@ -429,7 +409,7 @@ sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } -#endif +#endif /* defined(__NR_mmap2) */ #ifdef HAVE_GNU_LIBC_VERSION_H #ifdef HAVE_GNU_GET_LIBC_VERSION @@ -540,7 +520,7 @@ sb_chmod(scmp_filter_ctx ctx, sandbox_cfg_t *filter) rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chmod), SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + log_err(LD_BUG,"(Sandbox) failed to add chmod syscall, received " "libseccomp error %d", rc); return rc; } @@ -565,7 +545,7 @@ sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter) rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chown), SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + log_err(LD_BUG,"(Sandbox) failed to add chown syscall, received " "libseccomp error %d", rc); return rc; } @@ -744,6 +724,25 @@ sb_socketpair(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#ifdef HAVE_KIST_SUPPORT + +#include <linux/sockios.h> + +static int +sb_ioctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), + SCMP_CMP(1, SCMP_CMP_EQ, SIOCOUTQNSD)); + if (rc) + return rc; + return 0; +} + +#endif /* defined(HAVE_KIST_SUPPORT) */ + /** * Function responsible for setting up the setsockopt syscall for * the seccomp filter sandbox. @@ -784,7 +783,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUFFORCE)); if (rc) return rc; -#endif +#endif /* defined(HAVE_SYSTEMD) */ #ifdef IP_TRANSPARENT rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), @@ -792,7 +791,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, IP_TRANSPARENT)); if (rc) return rc; -#endif +#endif /* defined(IP_TRANSPARENT) */ #ifdef IPV6_V6ONLY rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), @@ -800,7 +799,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, IPV6_V6ONLY)); if (rc) return rc; -#endif +#endif /* defined(IPV6_V6ONLY) */ return 0; } @@ -833,7 +832,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUF)); if (rc) return rc; -#endif +#endif /* defined(HAVE_SYSTEMD) */ #ifdef HAVE_LINUX_NETFILTER_IPV4_H rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), @@ -841,7 +840,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, SO_ORIGINAL_DST)); if (rc) return rc; -#endif +#endif /* defined(HAVE_LINUX_NETFILTER_IPV4_H) */ #ifdef HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), @@ -849,7 +848,16 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, IP6T_SO_ORIGINAL_DST)); if (rc) return rc; -#endif +#endif /* defined(HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H) */ + +#ifdef HAVE_KIST_SUPPORT +#include <netinet/tcp.h> + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_TCP), + SCMP_CMP(2, SCMP_CMP_EQ, TCP_INFO)); + if (rc) + return rc; +#endif /* defined(HAVE_KIST_SUPPORT) */ return 0; } @@ -889,7 +897,7 @@ sb_fcntl64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } -#endif +#endif /* defined(__NR_fcntl64) */ /** * Function responsible for setting up the epoll_ctl syscall for @@ -1086,8 +1094,8 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(stat64), SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " - "libseccomp error %d", rc); + log_err(LD_BUG,"(Sandbox) failed to add stat64 syscall, received " + "libseccomp error %d", rc); return rc; } } @@ -1095,7 +1103,7 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } -#endif +#endif /* defined(__NR_stat64) */ static int sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter) @@ -1117,9 +1125,6 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter) static sandbox_filter_func_t filter_func[] = { sb_rt_sigaction, sb_rt_sigprocmask, -#if 0 - sb_execve, -#endif sb_time, sb_accept4, #ifdef __NR_mmap2 @@ -1148,6 +1153,9 @@ static sandbox_filter_func_t filter_func[] = { sb_setsockopt, sb_getsockopt, sb_socketpair, +#ifdef HAVE_KIST_SUPPORT + sb_ioctl, +#endif sb_kill }; @@ -1374,10 +1382,6 @@ sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_stat, file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1391,10 +1395,6 @@ sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(open), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1408,10 +1408,6 @@ sandbox_cfg_allow_chmod_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(chmod), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1425,10 +1421,6 @@ sandbox_cfg_allow_chown_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(chown), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1443,11 +1435,6 @@ sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2) elem = new_element2(SCMP_SYS(rename), file1, file2); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } - elem->next = *cfg; *cfg = elem; @@ -1460,10 +1447,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(openat), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1471,26 +1454,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) return 0; } -#if 0 -int -sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) -{ - sandbox_cfg_t *elem = NULL; - - elem = new_element(SCMP_SYS(execve), com); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } - - elem->next = *cfg; - *cfg = elem; - - return 0; -} - -#endif - /** Cache entry for getaddrinfo results; used when sandboxing is implemented * so that we can consult the cache when the sandbox prevents us from doing * getaddrinfo. @@ -1614,7 +1577,7 @@ sandbox_getaddrinfo(const char *name, const char *servname, return err; } - /* Otherwise, the sanbox is on. If we have an item, yield its cached + /* Otherwise, the sandbox is on. If we have an item, yield its cached result. */ if (item) { *res = item->res; @@ -1747,7 +1710,9 @@ install_syscall_filter(sandbox_cfg_t* cfg) // loading the seccomp2 filter if ((rc = seccomp_load(ctx))) { - log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)!", rc, + log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)! " + "Are you sure that your kernel has seccomp2 support? The " + "sandbox won't work without it.", rc, strerror(-rc)); goto end; } @@ -1815,7 +1780,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context) /* Clean up the top stack frame so we get the real function * name for the most recently failing function. */ clean_backtrace(syscall_cb_buf, depth, ctx); -#endif +#endif /* defined(USE_BACKTRACE) */ syscall_name = get_syscall_name(syscall); @@ -1889,7 +1854,7 @@ register_cfg(sandbox_cfg_t* cfg) return 0; } -#endif // USE_LIBSECCOMP +#endif /* defined(USE_LIBSECCOMP) */ #ifdef USE_LIBSECCOMP /** @@ -1919,7 +1884,7 @@ sandbox_is_active(void) { return sandbox_active != 0; } -#endif // USE_LIBSECCOMP +#endif /* defined(USE_LIBSECCOMP) */ sandbox_cfg_t* sandbox_cfg_new(void) @@ -1947,7 +1912,7 @@ sandbox_init(sandbox_cfg_t *cfg) "Currently, sandboxing is only implemented on Linux. The feature " "is disabled on your platform."); return 0; -#endif +#endif /* defined(USE_LIBSECCOMP) || ... */ } #ifndef USE_LIBSECCOMP @@ -1965,15 +1930,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) return 0; } -#if 0 -int -sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) -{ - (void)cfg; (void)com; - return 0; -} -#endif - int sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) { @@ -2012,5 +1968,5 @@ void sandbox_disable_getaddrinfo_cache(void) { } -#endif +#endif /* !defined(USE_LIBSECCOMP) */ diff --git a/src/common/sandbox.h b/src/common/sandbox.h index c5963e3119..d0f85570f4 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,7 +23,7 @@ */ #define SYS_SECCOMP 1 -#endif +#endif /* !defined(SYS_SECCOMP) */ #if defined(HAVE_SECCOMP_H) && defined(__linux__) #define USE_LIBSECCOMP @@ -101,7 +101,7 @@ typedef struct { sandbox_cfg_t *filter_dynamic; } sandbox_t; -#endif // USE_LIBSECCOMP +#endif /* defined(USE_LIBSECCOMP) */ #ifdef USE_LIBSECCOMP /** Pre-calls getaddrinfo in order to pre-record result. */ @@ -114,7 +114,7 @@ int sandbox_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void sandbox_freeaddrinfo(struct addrinfo *addrinfo); void sandbox_free_getaddrinfo_cache(void); -#else +#else /* !(defined(USE_LIBSECCOMP)) */ #define sandbox_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) #define sandbox_add_addrinfo(name) \ @@ -122,16 +122,16 @@ void sandbox_free_getaddrinfo_cache(void); #define sandbox_freeaddrinfo(addrinfo) \ freeaddrinfo((addrinfo)) #define sandbox_free_getaddrinfo_cache() -#endif +#endif /* defined(USE_LIBSECCOMP) */ #ifdef USE_LIBSECCOMP /** Returns a registered protected string used with the sandbox, given that * it matches the parameter. */ const char* sandbox_intern_string(const char *param); -#else +#else /* !(defined(USE_LIBSECCOMP)) */ #define sandbox_intern_string(s) (s) -#endif +#endif /* defined(USE_LIBSECCOMP) */ /** Creates an empty sandbox configuration file.*/ sandbox_cfg_t * sandbox_cfg_new(void); @@ -156,14 +156,6 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2); */ int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file); -#if 0 -/** - * Function used to add a execve allowed filename to a supplied configuration. - * The (char*) specifies the path to the allowed file; that pointer is stolen. - */ -int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com); -#endif - /** * Function used to add a stat/stat64 allowed filename to a configuration. * The (char*) specifies the path to the allowed file; that pointer is stolen. @@ -178,5 +170,5 @@ int sandbox_is_active(void); void sandbox_disable_getaddrinfo_cache(void); -#endif /* SANDBOX_H_ */ +#endif /* !defined(SANDBOX_H_) */ diff --git a/src/common/storagedir.c b/src/common/storagedir.c new file mode 100644 index 0000000000..c471ea911f --- /dev/null +++ b/src/common/storagedir.c @@ -0,0 +1,586 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "container.h" +#include "compat.h" +#include "confline.h" +#include "memarea.h" +#include "sandbox.h" +#include "storagedir.h" +#include "torlog.h" +#include "util.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#define FNAME_MIN_NUM 1000 + +/** A storage_dir_t represents a directory full of similar cached + * files. Filenames are decimal integers. Files can be cleaned as needed + * to limit total disk usage. */ +struct storage_dir_t { + /** Directory holding the files for this storagedir. */ + char *directory; + /** Either NULL, or a directory listing of the directory (as a smartlist + * of strings */ + smartlist_t *contents; + /** The largest number of non-temporary files we'll place in the + * directory. */ + int max_files; + /** If true, then 'usage' has been computed. */ + int usage_known; + /** The total number of bytes used in this directory */ + uint64_t usage; +}; + +/** Create or open a new storage directory at <b>dirname</b>, with + * capacity for up to <b>max_files</b> files. + */ +storage_dir_t * +storage_dir_new(const char *dirname, int max_files) +{ + if (check_private_dir(dirname, CPD_CREATE, NULL) < 0) + return NULL; + + storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t)); + d->directory = tor_strdup(dirname); + d->max_files = max_files; + return d; +} + +/** + * Drop all in-RAM storage for <b>d</b>. Does not delete any files. + */ +void +storage_dir_free(storage_dir_t *d) +{ + if (d == NULL) + return; + tor_free(d->directory); + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + tor_free(d); +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that <b>d</b> will need. + * + * The presence of this function is why we need an upper limit on the + * number of files in a storage_dir_t: we need to approve file operations + * one by one. + */ +int +storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg) +{ + int problems = 0; + int idx; + for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) { + char *path = NULL, *tmppath = NULL; + tor_asprintf(&path, "%s/%d", d->directory, idx); + tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx); + + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_rename(cfg, + tor_strdup(tmppath), tor_strdup(path)); + + tor_free(path); + tor_free(tmppath); + } + + return problems ? -1 : 0; +} + +/** + * Remove all files in <b>d</b> whose names end with ".tmp". + * + * Requires that the contents field of <b>d</b> is set. + */ +static void +storage_dir_clean_tmpfiles(storage_dir_t *d) +{ + if (!d->contents) + return; + SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) { + if (strcmpend(fname, ".tmp")) + continue; + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + if (unlink(sandbox_intern_string(path))) { + log_warn(LD_FS, "Unable to unlink %s while cleaning " + "temporary files: %s", escaped(path), strerror(errno)); + tor_free(path); + continue; + } + tor_free(path); + SMARTLIST_DEL_CURRENT(d->contents, fname); + tor_free(fname); + } SMARTLIST_FOREACH_END(fname); + + d->usage_known = 0; +} + +/** + * Re-scan the directory <b>d</b> to learn its contents. + */ +static int +storage_dir_rescan(storage_dir_t *d) +{ + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + d->usage = 0; + d->usage_known = 0; + if (NULL == (d->contents = tor_listdir(d->directory))) { + return -1; + } + storage_dir_clean_tmpfiles(d); + return 0; +} + +/** + * Return a smartlist containing the filenames within <b>d</b>. + */ +const smartlist_t * +storage_dir_list(storage_dir_t *d) +{ + if (! d->contents) + storage_dir_rescan(d); + return d->contents; +} + +/** + * Return the total number of bytes used for storage in <b>d</b>. + */ +uint64_t +storage_dir_get_usage(storage_dir_t *d) +{ + if (d->usage_known) + return d->usage; + + uint64_t total = 0; + SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) { + char *path = NULL; + struct stat st; + tor_asprintf(&path, "%s/%s", d->directory, cp); + if (stat(sandbox_intern_string(path), &st) == 0) { + total += st.st_size; + } + tor_free(path); + } SMARTLIST_FOREACH_END(cp); + + d->usage = total; + d->usage_known = 1; + return d->usage; +} + +/** Mmap a specified file within <b>d</b>. + * + * On failure, return NULL and set errno as for tor_mmap_file(). */ +tor_mmap_t * +storage_dir_map(storage_dir_t *d, const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + tor_mmap_t *result = tor_mmap_file(path); + int errval = errno; + tor_free(path); + if (result == NULL) + errno = errval; + return result; +} + +/** Read a file within <b>d</b> into a newly allocated buffer. Set + * *<b>sz_out</b> to its size. */ +uint8_t * +storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out) +{ + const int flags = bin ? RFTS_BIN : 0; + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + struct stat st; + char *contents = read_file_to_str(path, flags, &st); + if (contents && sz_out) { + // it fits in RAM, so we know its size is less than SIZE_MAX +#if UINT64_MAX > SIZE_MAX + tor_assert((uint64_t)st.st_size <= SIZE_MAX); +#endif + *sz_out = (size_t) st.st_size; + } + + tor_free(path); + return (uint8_t *) contents; +} + +/** Helper: Find an unused filename within the directory */ +static char * +find_unused_fname(storage_dir_t *d) +{ + if (!d->contents) { + if (storage_dir_rescan(d) < 0) + return NULL; + } + + char buf[16]; + int i; + /* Yuck; this is quadratic. Fortunately, that shouldn't matter much, + * since disk writes are more expensive by a lot. */ + for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) { + tor_snprintf(buf, sizeof(buf), "%d", i); + if (!smartlist_contains_string(d->contents, buf)) { + return tor_strdup(buf); + } + } + return NULL; +} + +/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of + * sized_chunk_t rather than a single byte array. */ +static int +storage_dir_save_chunks_to_file(storage_dir_t *d, + const smartlist_t *chunks, + int binary, + char **fname_out) +{ + uint64_t total_length = 0; + char *fname = find_unused_fname(d); + if (!fname) + return -1; + + SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch, + total_length += ch->len); + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + + int r = write_chunks_to_file(path, chunks, binary, 0); + if (r == 0) { + if (d->usage_known) + d->usage += total_length; + if (fname_out) { + *fname_out = tor_strdup(fname); + } + if (d->contents) + smartlist_add(d->contents, tor_strdup(fname)); + } + tor_free(fname); + tor_free(path); + return r; +} + +/** Try to write the <b>length</b> bytes at <b>data</b> into a new file + * in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a + * newly allocated string containing the filename. On failure, return + * -1. */ +int +storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out) +{ + smartlist_t *chunks = smartlist_new(); + sized_chunk_t chunk = { (const char *)data, length }; + smartlist_add(chunks, &chunk); + int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out); + smartlist_free(chunks); + return r; +} + +/** + * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string + * <b>str</b>. + */ +int +storage_dir_save_string_to_file(storage_dir_t *d, + const char *str, + int binary, + char **fname_out) +{ + return storage_dir_save_bytes_to_file(d, + (const uint8_t*)str, strlen(str), binary, fname_out); +} + +/** + * As storage_dir_save_bytes_to_file, but associates the data with the + * key-value pairs in <b>labels</b>. Files stored in this format can be + * recovered with storage_dir_map_labeled() or storage_dir_read_labeled(). + */ +int +storage_dir_save_labeled_to_file(storage_dir_t *d, + const config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out) +{ + /* + * The storage format is to prefix the data with the key-value pairs in + * <b>labels</b>, and a single NUL separator. But code outside this module + * MUST NOT rely on that format. + */ + + smartlist_t *chunks = smartlist_new(); + memarea_t *area = memarea_new(); + const config_line_t *line; + for (line = labels; line; line = line->next) { + sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t)); + sz->len = strlen(line->key) + 1 + strlen(line->value) + 1; + const size_t allocated = sz->len + 1; + char *bytes = memarea_alloc(area, allocated); + tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value); + sz->bytes = bytes; + smartlist_add(chunks, sz); + } + + sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t)); + nul->len = 1; + nul->bytes = "\0"; + smartlist_add(chunks, nul); + + sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t)); + datachunk->bytes = (const char *)data; + datachunk->len = length; + smartlist_add(chunks, datachunk); + + int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out); + smartlist_free(chunks); + memarea_drop_all(area); + return r; +} + +/** + * Map a file that was created with storage_dir_save_labeled_to_file(). On + * failure, return NULL. On success, write a set of newly allocated labels + * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and + * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t + * object whose contents should not be used -- it needs to be kept around, + * though, for as long as <b>data_out</b> is going to be valid. + * + * On failure, set errno as for tor_mmap_file() if the file was missing or + * empty, and set errno to EINVAL if the file was not in the labeled + * format expected. + */ +tor_mmap_t * +storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + const uint8_t **data_out, + size_t *sz_out) +{ + tor_mmap_t *m = storage_dir_map(dir, fname); + int errval; + if (! m) { + errval = errno; + goto err; + } + const char *nulp = memchr(m->data, '\0', m->size); + if (! nulp) { + errval = EINVAL; + goto err; + } + if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) { + errval = EINVAL; + goto err; + } + size_t offset = nulp - m->data + 1; + tor_assert(offset <= m->size); + *data_out = (const uint8_t *)(m->data + offset); + *sz_out = m->size - offset; + + return m; + err: + tor_munmap_file(m); + errno = errval; + return NULL; +} + +/** As storage_dir_map_labeled, but return a new byte array containing the + * data. */ +uint8_t * +storage_dir_read_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + size_t *sz_out) +{ + const uint8_t *data = NULL; + tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out, + &data, sz_out); + if (m == NULL) + return NULL; + uint8_t *result = tor_memdup(data, *sz_out); + tor_munmap_file(m); + return result; +} + +/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>. + * This function is a no-op if <b>d->usage_known</b> is 0. */ +static void +storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size) +{ + if (d->usage_known) { + if (! BUG(d->usage < removed_file_size)) { + /* This bug can also be triggered if an external process resized a file + * between the call to storage_dir_get_usage() that last checked + * actual usage (rather than relaying on cached usage), and the call to + * this function. */ + d->usage -= removed_file_size; + } else { + /* If we underflowed the cached directory size, re-check the sizes of all + * the files in the directory. This makes storage_dir_shrink() quadratic, + * but only if a process is continually changing file sizes in the + * storage directory (in which case, we have bigger issues). + * + * We can't just reset usage_known, because storage_dir_shrink() relies + * on knowing the usage. */ + storage_dir_rescan(d); + (void)storage_dir_get_usage(d); + } + } +} + +/** + * Remove the file called <b>fname</b> from <b>d</b>. + */ +void +storage_dir_remove_file(storage_dir_t *d, + const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + const char *ipath = sandbox_intern_string(path); + + uint64_t size = 0; + if (d->usage_known) { + struct stat st; + if (stat(ipath, &st) == 0) { + size = st.st_size; + } + } + if (unlink(ipath) == 0) { + storage_dir_reduce_usage(d, size); + } else { + log_warn(LD_FS, "Unable to unlink %s while removing file: %s", + escaped(path), strerror(errno)); + tor_free(path); + return; + } + if (d->contents) { + smartlist_string_remove(d->contents, fname); + } + + tor_free(path); +} + +/** Helper type: used to sort the members of storage directory by mtime. */ +typedef struct shrinking_dir_entry_t { + time_t mtime; + uint64_t size; + char *path; +} shrinking_dir_entry_t; + +/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */ +static int +shrinking_dir_entry_compare(const void *a_, const void *b_) +{ + const shrinking_dir_entry_t *a = a_; + const shrinking_dir_entry_t *b = b_; + + if (a->mtime < b->mtime) + return -1; + else if (a->mtime > b->mtime) + return 1; + else + return 0; +} + +/** + * Try to free space by removing the oldest files in <b>d</b>. Delete + * until no more than <b>target_size</b> bytes are left, and at least + * <b>min_to_remove</b> files have been removed... or until there is + * nothing left to remove. + * + * Return 0 on success; -1 on failure. + */ +int +storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove) +{ + if (d->usage_known && d->usage <= target_size && !min_to_remove) { + /* Already small enough. */ + return 0; + } + + if (storage_dir_rescan(d) < 0) + return -1; + + const uint64_t orig_usage = storage_dir_get_usage(d); + if (orig_usage <= target_size && !min_to_remove) { + /* Okay, small enough after rescan! */ + return 0; + } + + const int n = smartlist_len(d->contents); + shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t)); + SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) { + shrinking_dir_entry_t *ent = &ents[fname_sl_idx]; + struct stat st; + tor_asprintf(&ent->path, "%s/%s", d->directory, fname); + if (stat(sandbox_intern_string(ent->path), &st) == 0) { + ent->mtime = st.st_mtime; + ent->size = st.st_size; + } + } SMARTLIST_FOREACH_END(fname); + + qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare); + + int idx = 0; + while ((d->usage > target_size || min_to_remove > 0) && idx < n) { + if (unlink(sandbox_intern_string(ents[idx].path)) == 0) { + storage_dir_reduce_usage(d, ents[idx].size); + --min_to_remove; + } + ++idx; + } + + for (idx = 0; idx < n; ++idx) { + tor_free(ents[idx].path); + } + tor_free(ents); + + storage_dir_rescan(d); + + return 0; +} + +/** Remove all files in <b>d</b>. */ +int +storage_dir_remove_all(storage_dir_t *d) +{ + return storage_dir_shrink(d, 0, d->max_files); +} + +/** + * Return the largest number of non-temporary files we're willing to + * store in <b>d</b>. + */ +int +storage_dir_get_max_files(storage_dir_t *d) +{ + return d->max_files; +} + diff --git a/src/common/storagedir.h b/src/common/storagedir.h new file mode 100644 index 0000000000..3de0afc361 --- /dev/null +++ b/src/common/storagedir.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_STORAGEDIR_H +#define TOR_STORAGEDIR_H + +typedef struct storage_dir_t storage_dir_t; +struct config_line_t; +struct sandbox_cfg_elem; + +storage_dir_t * storage_dir_new(const char *dirname, int n_files); +void storage_dir_free(storage_dir_t *d); +int storage_dir_register_with_sandbox(storage_dir_t *d, + struct sandbox_cfg_elem **cfg); +const smartlist_t *storage_dir_list(storage_dir_t *d); +uint64_t storage_dir_get_usage(storage_dir_t *d); +tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname); +uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin, + size_t *sz_out); +int storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out); +int storage_dir_save_string_to_file(storage_dir_t *d, + const char *data, + int binary, + char **fname_out); +int storage_dir_save_labeled_to_file(storage_dir_t *d, + const struct config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out); +tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + struct config_line_t **labels_out, + const uint8_t **data_out, + size_t *size_out); +uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname, + struct config_line_t **labels_out, + size_t *sz_out); +void storage_dir_remove_file(storage_dir_t *d, + const char *fname); +int storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove); +int storage_dir_remove_all(storage_dir_t *d); +int storage_dir_get_max_files(storage_dir_t *d); + +#endif /* !defined(TOR_STORAGEDIR_H) */ + diff --git a/src/common/testsupport.h b/src/common/testsupport.h index 9ad2ba77e0..a3f2ff91ed 100644 --- a/src/common/testsupport.h +++ b/src/common/testsupport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TESTSUPPORT_H @@ -10,7 +10,7 @@ #else #define STATIC static #define EXTERN(type, name) -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Quick and dirty macros to implement test mocking. * @@ -76,15 +76,15 @@ do { \ func = func ##__real; \ } while (0) -#else +#else /* !(defined(TOR_UNIT_TESTS)) */ #define MOCK_DECL(rv, funcname, arglist) \ rv funcname arglist #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ rv funcname arglist attr #define MOCK_IMPL(rv, funcname, arglist) \ rv funcname arglist -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** @} */ -#endif +#endif /* !defined(TOR_TESTSUPPORT_H) */ diff --git a/src/common/timers.c b/src/common/timers.c index 41b2008ac4..c8e09414f4 100644 --- a/src/common/timers.c +++ b/src/common/timers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,6 +29,8 @@ #include "orconfig.h" +#define TOR_TIMERS_PRIVATE + #include "compat.h" #include "compat_libevent.h" #include "timers.h" @@ -51,7 +53,7 @@ struct timeout_cb { #else /* We're not exposing any of the functions outside this file. */ #define TIMEOUT_PUBLIC static -#endif +#endif /* defined(__GNUC__) */ /* We're not using periodic events. */ #define TIMEOUT_DISABLE_INTERVALS /* We always know the global_timeouts object, so we don't need each timeout @@ -148,6 +150,21 @@ libevent_timer_reschedule(void) event_add(global_timer_event, &d); } +/** Run the callback of every timer that has expired, based on the current + * output of monotime_get(). */ +STATIC void +timers_run_pending(void) +{ + monotime_t now; + monotime_get(&now); + timer_advance_to_cur_time(&now); + + tor_timer_t *t; + while ((t = timeouts_get(global_timeouts))) { + t->callback.cb(t, t->callback.arg, &now); + } +} + /** * Invoked when the libevent timer has expired: see which tor_timer_t events * have fired, activate their callbacks, and reschedule the libevent timer. @@ -159,14 +176,7 @@ libevent_timer_callback(evutil_socket_t fd, short what, void *arg) (void)what; (void)arg; - monotime_t now; - monotime_get(&now); - timer_advance_to_cur_time(&now); - - tor_timer_t *t; - while ((t = timeouts_get(global_timeouts))) { - t->callback.cb(t, t->callback.arg, &now); - } + timers_run_pending(); libevent_timer_reschedule(); } @@ -181,7 +191,7 @@ timers_initialize(void) if (BUG(global_timeouts)) return; // LCOV_EXCL_LINE - timeout_error_t err; + timeout_error_t err = 0; global_timeouts = timeouts_open(0, &err); if (!global_timeouts) { // LCOV_EXCL_START -- this can only fail on malloc failure. @@ -255,6 +265,20 @@ timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg) } /** + * Set *<b>cb_out</b> (if provided) to this timer's callback function, + * and *<b>arg_out</b> (if provided) to this timer's callback argument. + */ +void +timer_get_cb(const tor_timer_t *t, + timer_cb_fn_t *cb_out, void **arg_out) +{ + if (cb_out) + *cb_out = t->callback.cb; + if (arg_out) + *arg_out = t->callback.arg; +} + +/** * Schedule the timer t to fire at the current time plus a delay of * <b>delay</b> microseconds. All times are relative to monotime_get(). */ diff --git a/src/common/timers.h b/src/common/timers.h index 5f918f8e15..d4d4fb00a9 100644 --- a/src/common/timers.h +++ b/src/common/timers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TIMERS_H @@ -13,6 +13,8 @@ typedef void (*timer_cb_fn_t)(tor_timer_t *, void *, const struct monotime_t *); tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg); void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg); +void timer_get_cb(const tor_timer_t *t, + timer_cb_fn_t *cb_out, void **arg_out); void timer_schedule(tor_timer_t *t, const struct timeval *delay); void timer_disable(tor_timer_t *t); void timer_free(tor_timer_t *t); @@ -20,5 +22,9 @@ void timer_free(tor_timer_t *t); void timers_initialize(void); void timers_shutdown(void); +#ifdef TOR_TIMERS_PRIVATE +STATIC void timers_run_pending(void); #endif +#endif /* !defined(TOR_TIMERS_H) */ + diff --git a/src/common/torgzip.c b/src/common/torgzip.c deleted file mode 100644 index c44399aa74..0000000000 --- a/src/common/torgzip.c +++ /dev/null @@ -1,586 +0,0 @@ -/* Copyright (c) 2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.c - * \brief A simple in-memory gzip implementation. - **/ - -#include "orconfig.h" - -#include <stdlib.h> -#include <stdio.h> -#include <assert.h> -#include <string.h> -#include "torint.h" - -#ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> -#endif - -#include "util.h" -#include "torlog.h" -#include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include <zlib.h> - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ -static inline int -method_bits(compress_method_t method, zlib_compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - -static inline int -get_memlevel(zlib_compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - -/** @{ */ -/* These macros define the maximum allowable compression factor. Anything of - * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to - * have an uncompression factor (uncompressed size:compressed size ratio) of - * any greater than MAX_UNCOMPRESSION_FACTOR. - * - * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to - * be small to limit the attack multiplier, but we also want it to be large - * enough so that no legitimate document --even ones we might invent in the - * future -- ever compresses by a factor of greater than - * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably - * large range of possible values. IMO, anything over 8 is probably safe; IMO - * anything under 50 is probably sufficient. - */ -#define MAX_UNCOMPRESSION_FACTOR 25 -#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) -/** @} */ - -/** Return true if uncompressing an input of size <b>in_size</b> to an input - * of size at least <b>size_out</b> looks like a compression bomb. */ -static int -is_compression_bomb(size_t size_in, size_t size_out) -{ - if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) - return 0; - - return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); -} - -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the method described in <b>method</b>. Store the - * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - */ -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - /* Falls through. */ - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#ifdef OPENBSD - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); - return -1; -} - -/** Given zero or more zlib-compressed or gzip-compressed strings of - * total length - * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated - * buffer, using the method described in <b>method</b>. Store the uncompressed - * string in *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on - * success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a - * failure; otherwise we decompress as much as we can. Warn about truncated - * or corrupt inputs at <b>protocol_warn_level</b>. - */ -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - /* Falls through. */ - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } - return -1; -} - -/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely - * to be compressed or not. If it is, return the likeliest compression method. - * Otherwise, return UNKNOWN_METHOD. - */ -compress_method_t -detect_compression_method(const char *in, size_t in_len) -{ - if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { - return GZIP_METHOD; - } else if (in_len > 2 && (in[0] & 0x0f) == 8 && - (ntohs(get_uint16(in)) % 31) == 0) { - return ZLIB_METHOD; - } else { - return UNKNOWN_METHOD; - } -} - -/** Internal state for an incremental zlib compression/decompression. The - * body of this struct is not exposed. */ -struct tor_zlib_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ - - /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; -}; - -/** Construct and return a tor_zlib_state_t object using <b>method</b>. If - * <b>compress</b>, it's for compression; otherwise it's for - * decompression. */ -tor_zlib_state_t * -tor_zlib_new(int compress_, compress_method_t method, - zlib_compression_level_t compression_level) -{ - tor_zlib_state_t *out; - int bits, memlevel; - - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } - - out = tor_malloc_zero(sizeof(tor_zlib_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - - total_zlib_allocation += out->allocation; - - return out; - - err: - tor_free(out); - return NULL; -} - -/** Compress/decompress some bytes using <b>state</b>. Read up to - * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes - * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, - * we've reached the end of the input. - * - * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. - * Return TOR_ZLIB_OK if we're processed everything from the input. - * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>. - * Return TOR_ZLIB_ERR if the stream is corrupt. - */ -tor_zlib_output_t -tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) -{ - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; - - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } - - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); - - *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; - - if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_ZLIB_ERR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_ZLIB_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_ZLIB_OK; - return TOR_ZLIB_BUF_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_ZLIB_BUF_FULL; - return TOR_ZLIB_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_ZLIB_ERR; - } -} - -/** Deallocate <b>state</b>. */ -void -tor_zlib_free(tor_zlib_state_t *state) -{ - if (!state) - return; - - total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); - - tor_free(state); -} - -/** Return an approximate number of bytes used in RAM to hold a state with - * window bits <b>windowBits</b> and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - -/** Return the approximate number of bytes allocated for <b>state</b>. */ -size_t -tor_zlib_state_size(const tor_zlib_state_t *state) -{ - return state->allocation; -} - -/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; -} - diff --git a/src/common/torgzip.h b/src/common/torgzip.h deleted file mode 100644 index 00f62dcb45..0000000000 --- a/src/common/torgzip.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.h - * \brief Headers for torgzip.h - **/ - -#ifndef TOR_TORGZIP_H -#define TOR_TORGZIP_H - -/** Enumeration of what kind of compression to use. Only ZLIB_METHOD is - * guaranteed to be supported by the compress/uncompress functions here; - * GZIP_METHOD may be supported if we built against zlib version 1.2 or later - * and is_gzip_supported() returns true. */ -typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 -} compress_method_t; - -/** - * Enumeration to define tradeoffs between memory usage and compression level. - * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most - * memory. - **/ -typedef enum { - HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} zlib_compression_level_t; - -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - -int is_gzip_supported(void); - -const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - -compress_method_t detect_compression_method(const char *in, size_t in_len); - -/** Return values from tor_zlib_process; see that function's documentation for - * details. */ -typedef enum { - TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR -} tor_zlib_output_t; -/** Internal state for an incremental zlib compression/decompression. */ -typedef struct tor_zlib_state_t tor_zlib_state_t; -tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - zlib_compression_level_t level); - -tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_zlib_free(tor_zlib_state_t *state); - -size_t tor_zlib_state_size(const tor_zlib_state_t *state); -size_t tor_zlib_get_total_allocation(void); - -#endif - diff --git a/src/common/torint.h b/src/common/torint.h index 58c30f41a8..bc81c114f8 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -34,8 +34,8 @@ does the same thing (but doesn't defined __FreeBSD__). */ #include <machine/limits.h> -#endif -#endif +#endif /* !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__) */ +#endif /* defined(HAVE_MACHINE_LIMITS_H) */ #ifdef HAVE_INTTYPES_H #include <inttypes.h> #endif @@ -80,7 +80,7 @@ typedef signed char int8_t; typedef unsigned char uint8_t; #define HAVE_UINT8_T #endif -#endif +#endif /* (SIZEOF_CHAR == 1) */ #if (SIZEOF_SHORT == 2) #ifndef HAVE_INT16_T @@ -91,7 +91,7 @@ typedef signed short int16_t; typedef unsigned short uint16_t; #define HAVE_UINT16_T #endif -#endif +#endif /* (SIZEOF_SHORT == 2) */ #if (SIZEOF_INT == 2) #ifndef HAVE_INT16_T @@ -129,7 +129,7 @@ typedef unsigned int uint32_t; #ifndef INT32_MIN #define INT32_MIN (-2147483647-1) #endif -#endif +#endif /* (SIZEOF_INT == 2) || ... */ #if (SIZEOF_LONG == 4) #ifndef HAVE_INT32_T @@ -142,7 +142,7 @@ typedef unsigned long uint32_t; #ifndef UINT32_MAX #define UINT32_MAX 0xfffffffful #endif -#endif +#endif /* !defined(HAVE_UINT32_T) */ #elif (SIZEOF_LONG == 8) #ifndef HAVE_INT64_T typedef signed long int64_t; @@ -155,7 +155,7 @@ typedef unsigned long uint64_t; #ifndef UINT64_MAX #define UINT64_MAX 0xfffffffffffffffful #endif -#endif +#endif /* (SIZEOF_LONG == 4) || ... */ #if (SIZEOF_LONG_LONG == 8) #ifndef HAVE_INT64_T @@ -172,7 +172,7 @@ typedef unsigned long long uint64_t; #ifndef INT64_MAX #define INT64_MAX 0x7fffffffffffffffll #endif -#endif +#endif /* (SIZEOF_LONG_LONG == 8) */ #if (SIZEOF___INT64 == 8) #ifndef HAVE_INT64_T @@ -189,7 +189,7 @@ typedef unsigned __int64 uint64_t; #ifndef INT64_MAX #define INT64_MAX 0x7fffffffffffffffi64 #endif -#endif +#endif /* (SIZEOF___INT64 == 8) */ #ifndef INT64_MIN #define INT64_MIN ((- INT64_MAX) - 1) @@ -202,8 +202,8 @@ typedef unsigned __int64 uint64_t; #define SIZE_MAX UINT32_MAX #else #error "Can't define SIZE_MAX" -#endif -#endif +#endif /* SIZEOF_SIZE_T == 8 || ... */ +#endif /* !defined(SIZE_MAX) */ #ifndef HAVE_SSIZE_T #if SIZEOF_SIZE_T == 8 @@ -212,8 +212,8 @@ typedef int64_t ssize_t; typedef int32_t ssize_t; #else #error "Can't define ssize_t." -#endif -#endif +#endif /* SIZEOF_SIZE_T == 8 || ... */ +#endif /* !defined(HAVE_SSIZE_T) */ #if (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8) #ifndef HAVE_INTPTR_T @@ -235,7 +235,7 @@ typedef uint32_t uintptr_t; #endif #else #error "void * is either >8 bytes or <= 2. In either case, I am confused." -#endif +#endif /* (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8) || ... */ #ifndef HAVE_INT8_T #error "Missing type int8_t" @@ -275,8 +275,8 @@ typedef uint32_t uintptr_t; #define LONG_MAX 0x7fffffffffffffffL #else #error "Can't define LONG_MAX" -#endif -#endif +#endif /* (SIZEOF_LONG == 4) || ... */ +#endif /* !defined(LONG_MAX) */ #ifndef INT_MAX #if (SIZEOF_INT == 4) @@ -285,8 +285,8 @@ typedef uint32_t uintptr_t; #define INT_MAX 0x7fffffffffffffffL #else #error "Can't define INT_MAX" -#endif -#endif +#endif /* (SIZEOF_INT == 4) || ... */ +#endif /* !defined(INT_MAX) */ #ifndef UINT_MAX #if (SIZEOF_INT == 2) @@ -297,8 +297,8 @@ typedef uint32_t uintptr_t; #define UINT_MAX 0xffffffffffffffffu #else #error "Can't define UINT_MAX" -#endif -#endif +#endif /* (SIZEOF_INT == 2) || ... */ +#endif /* !defined(UINT_MAX) */ #ifndef SHORT_MAX #if (SIZEOF_SHORT == 2) @@ -307,8 +307,8 @@ typedef uint32_t uintptr_t; #define SHORT_MAX 0x7fffffff #else #error "Can't define SHORT_MAX" -#endif -#endif +#endif /* (SIZEOF_SHORT == 2) || ... */ +#endif /* !defined(SHORT_MAX) */ #ifndef TIME_MAX @@ -320,9 +320,9 @@ typedef uint32_t uintptr_t; #define TIME_MAX ((time_t)INT64_MAX) #else #error "Can't define TIME_MAX" -#endif +#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */ -#endif /* ifndef(TIME_MAX) */ +#endif /* !defined(TIME_MAX) */ #ifndef TIME_MIN @@ -334,9 +334,9 @@ typedef uint32_t uintptr_t; #define TIME_MIN ((time_t)INT64_MIN) #else #error "Can't define TIME_MIN" -#endif +#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */ -#endif /* ifndef(TIME_MIN) */ +#endif /* !defined(TIME_MIN) */ #ifndef SIZE_MAX #if (SIZEOF_SIZE_T == 4) @@ -345,8 +345,8 @@ typedef uint32_t uintptr_t; #define SIZE_MAX UINT64_MAX #else #error "Can't define SIZE_MAX" -#endif -#endif +#endif /* (SIZEOF_SIZE_T == 4) || ... */ +#endif /* !defined(SIZE_MAX) */ #ifndef SSIZE_MAX #if (SIZEOF_SIZE_T == 4) @@ -355,13 +355,13 @@ typedef uint32_t uintptr_t; #define SSIZE_MAX INT64_MAX #else #error "Can't define SSIZE_MAX" -#endif -#endif +#endif /* (SIZEOF_SIZE_T == 4) || ... */ +#endif /* !defined(SSIZE_MAX) */ /** Any ssize_t larger than this amount is likely to be an underflow. */ #define SSIZE_T_CEILING ((ssize_t)(SSIZE_MAX-16)) /** Any size_t larger than this amount is likely to be an underflow. */ #define SIZE_T_CEILING ((size_t)(SSIZE_MAX-16)) -#endif /* __TORINT_H */ +#endif /* !defined(TOR_TORINT_H) */ diff --git a/src/common/torlog.h b/src/common/torlog.h index 20b7d938f0..be24b2b908 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,7 +22,7 @@ #error "Your syslog.h thinks high numbers are more important. " \ "We aren't prepared to deal with that." #endif -#else +#else /* !(defined(HAVE_SYSLOG_H)) */ /* Note: Syslog's logging code refers to priorities, with 0 being the most * important. Thus, all our comparisons needed to be reversed when we added * syslog support. @@ -48,7 +48,7 @@ /** Error-level severity: for messages that only appear when something has gone * very wrong, and the Tor process can no longer proceed. */ #define LOG_ERR 3 -#endif +#endif /* defined(HAVE_SYSLOG_H) */ /* Logging domains */ @@ -99,10 +99,14 @@ #define LD_CHANNEL (1u<<21) /** Scheduler */ #define LD_SCHED (1u<<22) +/** Guard nodes */ +#define LD_GUARD (1u<<23) +/** Generation and application of consensus diffs. */ +#define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ -#define LD_DOS (1u<<23) +#define LD_DOS (1u<<25) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 24 +#define N_LOGGING_DOMAINS 26 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ @@ -211,7 +215,7 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, #define log_err(domain, args...) \ log_fn_(LOG_ERR, domain, __FUNCTION__, args) -#else /* ! defined(__GNUC__) */ +#else /* !(defined(__GNUC__) && __GNUC__ <= 3) */ /* Here are the c99 variadic macros, to work with non-GCC compilers */ @@ -238,7 +242,7 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, #define log_fn_ratelim(ratelim, severity, domain, args,...) \ log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, \ args, ##__VA_ARGS__) -#endif +#endif /* defined(__GNUC__) && __GNUC__ <= 3 */ #ifdef LOG_PRIVATE MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, @@ -247,5 +251,5 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, #endif # define TOR_TORLOG_H -#endif +#endif /* !defined(TOR_TORLOG_H) */ diff --git a/src/common/tortls.c b/src/common/tortls.c index a4e188603c..e8c51879bd 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,6 +17,7 @@ #include "orconfig.h" #define TORTLS_PRIVATE +#define TORTLS_OPENSSL_PRIVATE #include <assert.h> #ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/ @@ -75,7 +76,7 @@ ENABLE_GCC_WARNING(redundant-decls) * SSL3 safely at the same time. */ #define DISABLE_SSL3_HANDSHAKE -#endif +#endif /* OPENSSL_VERSION_NUMBER < OPENSSL_V(1,0,0,'f') */ /* We redefine these so that we can run correctly even if the vendor gives us * a version of OpenSSL that does not match its header files. (Apple: I am @@ -136,6 +137,7 @@ static void tor_tls_context_decref(tor_tls_context_t *ctx); static void tor_tls_context_incref(tor_tls_context_t *ctx); static int check_cert_lifetime_internal(int severity, const X509 *cert, + time_t now, int past_tolerance, int future_tolerance); /** Global TLS contexts. We keep them here because nobody else needs @@ -388,7 +390,7 @@ tor_tls_init(void) "when configuring it) would make ECDH much faster."); } /* LCOV_EXCL_STOP */ -#endif +#endif /* (SIZEOF_VOID_P >= 8 && ... */ tor_tls_allocate_tor_tls_object_ex_data_index(); @@ -442,8 +444,9 @@ tor_x509_name_new(const char *cname) goto error; /* LCOV_EXCL_BR_STOP */ return name; - error: + /* LCOV_EXCL_START : these lines will only execute on out of memory errors*/ + error: X509_NAME_free(name); return NULL; /* LCOV_EXCL_STOP */ @@ -458,11 +461,11 @@ tor_x509_name_new(const char *cname) * Return a certificate on success, NULL on failure. */ MOCK_IMPL(STATIC X509 *, - tor_tls_create_certificate,(crypto_pk_t *rsa, - crypto_pk_t *rsa_sign, - const char *cname, - const char *cname_sign, - unsigned int cert_lifetime)) +tor_tls_create_certificate,(crypto_pk_t *rsa, + crypto_pk_t *rsa_sign, + const char *cname, + const char *cname_sign, + unsigned int cert_lifetime)) { /* OpenSSL generates self-signed certificates with random 64-bit serial * numbers, so let's do that too. */ @@ -482,8 +485,25 @@ MOCK_IMPL(STATIC X509 *, * then we might pick a time where we're about to expire. Lastly, be * sure to start on a day boundary. */ time_t now = time(NULL); - start_time = crypto_rand_time_range(now - cert_lifetime, now) + 2*24*3600; - start_time -= start_time % (24*3600); + /* Our certificate lifetime will be cert_lifetime no matter what, but if we + * start cert_lifetime in the past, we'll have 0 real lifetime. instead we + * start up to (cert_lifetime - min_real_lifetime - start_granularity) in + * the past. */ + const time_t min_real_lifetime = 24*3600; + const time_t start_granularity = 24*3600; + time_t earliest_start_time; + /* Don't actually start in the future! */ + if (cert_lifetime <= min_real_lifetime + start_granularity) { + earliest_start_time = now - 1; + } else { + earliest_start_time = now + min_real_lifetime + start_granularity + - cert_lifetime; + } + start_time = crypto_rand_time_range(earliest_start_time, now); + /* Round the start time back to the start of a day. */ + start_time -= start_time % start_granularity; + + end_time = start_time + cert_lifetime; tor_assert(rsa); tor_assert(cname); @@ -517,12 +537,12 @@ MOCK_IMPL(STATIC X509 *, if (!X509_time_adj(X509_get_notBefore(x509),0,&start_time)) goto error; - end_time = start_time + cert_lifetime; if (!X509_time_adj(X509_get_notAfter(x509),0,&end_time)) goto error; if (!X509_set_pubkey(x509, pkey)) goto error; - if (!X509_sign(x509, sign_pkey, EVP_sha1())) + + if (!X509_sign(x509, sign_pkey, EVP_sha256())) goto error; goto done; @@ -605,6 +625,12 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 ":" #endif +#ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_CCM + TLS1_TXT_DHE_RSA_WITH_AES_256_CCM ":" +#endif +#ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_CCM + TLS1_TXT_DHE_RSA_WITH_AES_128_CCM ":" +#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 ":" #endif @@ -614,8 +640,14 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /* Required */ TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" /* Required */ - TLS1_TXT_DHE_RSA_WITH_AES_128_SHA - ; + TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":" +#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 + TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 ":" +#endif +#ifdef TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 + TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 +#endif + ; /* Note: to set up your own private testing network with link crypto * disabled, set your Tors' cipher list to @@ -656,7 +688,7 @@ tor_x509_cert_free(tor_x509_cert_t *cert) * Steals a reference to x509_cert. */ MOCK_IMPL(STATIC tor_x509_cert_t *, - tor_x509_cert_new,(X509 *x509_cert)) +tor_x509_cert_new,(X509 *x509_cert)) { tor_x509_cert_t *cert; EVP_PKEY *pkey; @@ -670,12 +702,7 @@ MOCK_IMPL(STATIC tor_x509_cert_t *, length = i2d_X509(x509_cert, &buf); cert = tor_malloc_zero(sizeof(tor_x509_cert_t)); if (length <= 0 || buf == NULL) { - /* LCOV_EXCL_START for the same reason as the exclusion above */ - tor_free(cert); - log_err(LD_CRYPTO, "Couldn't get length of encoded x509 certificate"); - X509_free(x509_cert); - return NULL; - /* LCOV_EXCL_STOP */ + goto err; } cert->encoded_len = (size_t) length; cert->encoded = tor_malloc(length); @@ -690,13 +717,25 @@ MOCK_IMPL(STATIC tor_x509_cert_t *, if ((pkey = X509_get_pubkey(x509_cert)) && (rsa = EVP_PKEY_get1_RSA(pkey))) { crypto_pk_t *pk = crypto_new_pk_from_rsa_(rsa); - crypto_pk_get_common_digests(pk, &cert->pkey_digests); + if (crypto_pk_get_common_digests(pk, &cert->pkey_digests) < 0) { + crypto_pk_free(pk); + EVP_PKEY_free(pkey); + goto err; + } + cert->pkey_digests_set = 1; crypto_pk_free(pk); EVP_PKEY_free(pkey); } return cert; + err: + /* LCOV_EXCL_START for the same reason as the exclusion above */ + tor_free(cert); + log_err(LD_CRYPTO, "Couldn't wrap encoded X509 certificate."); + X509_free(x509_cert); + return NULL; + /* LCOV_EXCL_STOP */ } /** Return a new copy of <b>cert</b>. */ @@ -800,8 +839,8 @@ tor_tls_context_decref(tor_tls_context_t *ctx) /** Set *<b>link_cert_out</b> and *<b>id_cert_out</b> to the link certificate * and ID certificate that we're currently using for our V3 in-protocol * handshake's certificate chain. If <b>server</b> is true, provide the certs - * that we use in server mode; otherwise, provide the certs that we use in - * client mode. */ + * that we use in server mode (auth, ID); otherwise, provide the certs that we + * use in client mode. (link, ID) */ int tor_tls_get_my_certs(int server, const tor_x509_cert_t **link_cert_out, @@ -831,7 +870,7 @@ tor_tls_get_my_client_auth_key(void) /** * Return a newly allocated copy of the public key that a certificate - * certifies. Return NULL if the cert's key is not RSA. + * certifies. Watch out! This returns NULL if the cert's key is not RSA. */ crypto_pk_t * tor_tls_cert_get_key(tor_x509_cert_t *cert) @@ -886,6 +925,7 @@ int tor_tls_cert_is_valid(int severity, const tor_x509_cert_t *cert, const tor_x509_cert_t *signing_cert, + time_t now, int check_rsa_1024) { check_no_tls_errors(); @@ -905,7 +945,7 @@ tor_tls_cert_is_valid(int severity, /* okay, the signature checked out right. Now let's check the check the * lifetime. */ - if (check_cert_lifetime_internal(severity, cert->cert, + if (check_cert_lifetime_internal(severity, cert->cert, now, 48*60*60, 30*24*60*60) < 0) goto bad; @@ -1050,6 +1090,8 @@ tor_tls_context_init_one(tor_tls_context_t **ppcontext, /** The group we should use for ecdhe when none was selected. */ #define NID_tor_default_ecdhe_group NID_X9_62_prime256v1 +#define RSA_LINK_KEY_BITS 2048 + /** Create a new TLS context for use with Tor TLS handshakes. * <b>identity</b> should be set to the identity key used to sign the * certificate. @@ -1075,7 +1117,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, /* Generate short-term RSA key for use with TLS. */ if (!(rsa = crypto_pk_new())) goto error; - if (crypto_pk_generate_key(rsa)<0) + if (crypto_pk_generate_key_with_bits(rsa, RSA_LINK_KEY_BITS)<0) goto error; if (!is_client) { /* Generate short-term RSA key for use in the in-protocol ("v3") @@ -1118,7 +1160,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, * with existing Tors. */ if (!(result->ctx = SSL_CTX_new(TLSv1_method()))) goto error; -#endif +#endif /* 0 */ /* Tell OpenSSL to use TLS 1.0 or later but not SSL2 or SSL3. */ #ifdef HAVE_TLS_METHOD @@ -1127,7 +1169,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, #else if (!(result->ctx = SSL_CTX_new(SSLv23_method()))) goto error; -#endif +#endif /* defined(HAVE_TLS_METHOD) */ SSL_CTX_set_options(result->ctx, SSL_OP_NO_SSLv2); SSL_CTX_set_options(result->ctx, SSL_OP_NO_SSLv3); @@ -1165,17 +1207,20 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_options(result->ctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); } + + /* Don't actually allow compression; it uses RAM and time, it makes TLS + * vulnerable to CRIME-style attacks, and most of the data we transmit over + * TLS is encrypted (and therefore uncompressible) anyway. */ #ifdef SSL_OP_NO_COMPRESSION SSL_CTX_set_options(result->ctx, SSL_OP_NO_COMPRESSION); #endif #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0) #ifndef OPENSSL_NO_COMP - /* Don't actually allow compression; it uses ram and time, but the data - * we transmit is all encrypted anyway. */ if (result->ctx->comp_methods) result->ctx->comp_methods = NULL; #endif -#endif +#endif /* OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0) */ + #ifdef SSL_MODE_RELEASE_BUFFERS SSL_CTX_set_mode(result->ctx, SSL_MODE_RELEASE_BUFFERS); #endif @@ -1335,7 +1380,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher); return c != NULL; } -#else +#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */ # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) if (m && m->get_cipher_by_char) { @@ -1349,7 +1394,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((c->id & 0xffff) == cipher); return c != NULL; } -# endif +#endif /* defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) */ # ifndef OPENSSL_1_1_API if (m && m->get_cipher && m->num_ciphers) { /* It would seem that some of the "let's-clean-up-openssl" forks have @@ -1365,12 +1410,12 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) } return 0; } -# endif +#endif /* !defined(OPENSSL_1_1_API) */ (void) ssl; (void) m; (void) cipher; return 1; /* No way to search */ -#endif +#endif /* defined(HAVE_SSL_CIPHER_FIND) */ } /** Remove from v2_cipher_list every cipher that we don't support, so that @@ -1498,7 +1543,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl) return CIPHERS_ERR; } ciphers = session->ciphers; -#endif +#endif /* defined(HAVE_SSL_GET_CLIENT_CIPHERS) */ return tor_tls_classify_client_ciphers(ssl, ciphers) >= CIPHERS_V2; } @@ -1629,7 +1674,7 @@ tor_tls_new(int sock, int isServer) SSL_set_tlsext_host_name(result->ssl, fake_hostname); tor_free(fake_hostname); } -#endif +#endif /* defined(SSL_set_tlsext_host_name) */ if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { @@ -1756,7 +1801,7 @@ tor_tls_assert_renegotiation_unblocked(tor_tls_t *tls) tor_assert(0 != (options & SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION)); #else (void) tls; -#endif +#endif /* defined(SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION) && ... */ } /** Return whether this tls initiated the connect (client) or @@ -2073,13 +2118,13 @@ tor_tls_get_own_cert,(tor_tls_t *tls)) /** Warn that a certificate lifetime extends through a certain range. */ static void -log_cert_lifetime(int severity, const X509 *cert, const char *problem) +log_cert_lifetime(int severity, const X509 *cert, const char *problem, + time_t now) { BIO *bio = NULL; BUF_MEM *buf; char *s1=NULL, *s2=NULL; char mytime[33]; - time_t now = time(NULL); struct tm tm; size_t n; @@ -2227,6 +2272,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity_key) */ int tor_tls_check_lifetime(int severity, tor_tls_t *tls, + time_t now, int past_tolerance, int future_tolerance) { X509 *cert; @@ -2235,7 +2281,7 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls, if (!(cert = SSL_get_peer_certificate(tls->ssl))) goto done; - if (check_cert_lifetime_internal(severity, cert, + if (check_cert_lifetime_internal(severity, cert, now, past_tolerance, future_tolerance) < 0) goto done; @@ -2251,30 +2297,48 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls, /** Helper: check whether <b>cert</b> is expired give or take * <b>past_tolerance</b> seconds, or not-yet-valid give or take - * <b>future_tolerance</b> seconds. If it is live, return 0. If it is not - * live, log a message and return -1. */ + * <b>future_tolerance</b> seconds. (Relative to the current time + * <b>now</b>.) If it is live, return 0. If it is not live, log a message + * and return -1. */ static int check_cert_lifetime_internal(int severity, const X509 *cert, + time_t now, int past_tolerance, int future_tolerance) { - time_t now, t; - - now = time(NULL); + time_t t; t = now + future_tolerance; if (X509_cmp_time(X509_get_notBefore_const(cert), &t) > 0) { - log_cert_lifetime(severity, cert, "not yet valid"); + log_cert_lifetime(severity, cert, "not yet valid", now); return -1; } t = now - past_tolerance; if (X509_cmp_time(X509_get_notAfter_const(cert), &t) < 0) { - log_cert_lifetime(severity, cert, "already expired"); + log_cert_lifetime(severity, cert, "already expired", now); return -1; } return 0; } +#ifdef TOR_UNIT_TESTS +/* Testing only: return a new x509 cert with the same contents as <b>inp</b>, + but with the expiration time <b>new_expiration_time</b>, signed with + <b>signing_key</b>. */ +STATIC tor_x509_cert_t * +tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key) +{ + X509 *newc = X509_dup(inp->cert); + X509_time_adj(X509_get_notAfter(newc), 0, &new_expiration_time); + EVP_PKEY *pk = crypto_pk_get_evp_pkey_(signing_key, 1); + tor_assert(X509_sign(newc, pk, EVP_sha256())); + EVP_PKEY_free(pk); + return tor_x509_cert_new(newc); +} +#endif /* defined(TOR_UNIT_TESTS) */ + /** Return the number of bytes available for reading from <b>tls</b>. */ int @@ -2317,10 +2381,10 @@ tor_tls_get_n_raw_bytes(tor_tls_t *tls, size_t *n_read, size_t *n_written) if (BIO_method_type(wbio) == BIO_TYPE_BUFFER && (tmpbio = BIO_next(wbio)) != NULL) wbio = tmpbio; -#else +#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5)) */ if (wbio->method == BIO_f_buffer() && (tmpbio = BIO_next(wbio)) != NULL) wbio = tmpbio; -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) */ w = (unsigned long) BIO_number_written(wbio); /* We are ok with letting these unsigned ints go "negative" here: @@ -2399,7 +2463,7 @@ SSL_get_client_random(SSL *s, uint8_t *out, size_t len) memcpy(out, s->s3->client_random, len); return len; } -#endif +#endif /* !defined(HAVE_SSL_GET_CLIENT_RANDOM) */ #ifndef HAVE_SSL_GET_SERVER_RANDOM static size_t @@ -2412,7 +2476,7 @@ SSL_get_server_random(SSL *s, uint8_t *out, size_t len) memcpy(out, s->s3->server_random, len); return len; } -#endif +#endif /* !defined(HAVE_SSL_GET_SERVER_RANDOM) */ #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY STATIC size_t @@ -2426,7 +2490,7 @@ SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, size_t len) memcpy(out, s->master_key, len); return len; } -#endif +#endif /* !defined(HAVE_SSL_SESSION_GET_MASTER_KEY) */ /** Set the DIGEST256_LEN buffer at <b>secrets_out</b> to the value used in * the v3 handshake to prove that the client knows the TLS secrets for the @@ -2493,6 +2557,28 @@ tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out)) return 0; } +/** Using the RFC5705 key material exporting construction, and the + * provided <b>context</b> (<b>context_len</b> bytes long) and + * <b>label</b> (a NUL-terminated string), compute a 32-byte secret in + * <b>secrets_out</b> that only the parties to this TLS session can + * compute. Return 0 on success and -1 on failure. + */ +MOCK_IMPL(int, +tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out, + const uint8_t *context, + size_t context_len, + const char *label)) +{ + tor_assert(tls); + tor_assert(tls->ssl); + + int r = SSL_export_keying_material(tls->ssl, + secrets_out, DIGEST256_LEN, + label, strlen(label), + context, context_len, 1); + return (r == 1) ? 0 : -1; +} + /** Examine the amount of memory used and available for buffers in <b>tls</b>. * Set *<b>rbuf_capacity</b> to the amount of storage allocated for the read * buffer and *<b>rbuf_bytes</b> to the amount actually used. @@ -2513,7 +2599,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls, (void)wbuf_bytes; return -1; -#else +#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)) */ if (tls->ssl->s3->rbuf.buf) *rbuf_capacity = tls->ssl->s3->rbuf.len; else @@ -2525,7 +2611,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls, *rbuf_bytes = tls->ssl->s3->rbuf.left; *wbuf_bytes = tls->ssl->s3->wbuf.left; return 0; -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ } /** Check whether the ECC group requested is supported by the current OpenSSL diff --git a/src/common/tortls.h b/src/common/tortls.h index f018c45c82..6145f7dbc9 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_H @@ -63,12 +63,17 @@ typedef enum { } tor_tls_state_t; #define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t) +struct x509_st; +struct ssl_st; +struct ssl_ctx_st; +struct ssl_session_st; + /** Holds a SSL_CTX object and related state used to configure TLS * connections. */ typedef struct tor_tls_context_t { int refcnt; - SSL_CTX *ctx; + struct ssl_ctx_st *ctx; tor_x509_cert_t *my_link_cert; tor_x509_cert_t *my_id_cert; tor_x509_cert_t *my_auth_cert; @@ -78,7 +83,7 @@ typedef struct tor_tls_context_t { /** Structure that we use for a single certificate. */ struct tor_x509_cert_t { - X509 *cert; + struct x509_st *cert; uint8_t *encoded; size_t encoded_len; unsigned pkey_digests_set : 1; @@ -92,7 +97,7 @@ struct tor_x509_cert_t { struct tor_tls_t { uint32_t magic; tor_tls_context_t *context; /** A link to the context object for this tls. */ - SSL *ssl; /**< An OpenSSL SSL object. */ + struct ssl_st *ssl; /**< An OpenSSL SSL object. */ int socket; /**< The underlying file descriptor for this TLS connection. */ char *address; /**< An address to log when describing this connection. */ tor_tls_state_bitfield_t state : 3; /**< The current SSL state, @@ -128,35 +133,45 @@ struct tor_tls_t { STATIC int tor_errno_to_tls_error(int e); STATIC int tor_tls_get_error(tor_tls_t *tls, int r, int extra, const char *doing, int severity, int domain); -STATIC tor_tls_t *tor_tls_get_by_ssl(const SSL *ssl); +STATIC tor_tls_t *tor_tls_get_by_ssl(const struct ssl_st *ssl); STATIC void tor_tls_allocate_tor_tls_object_ex_data_index(void); +#ifdef TORTLS_OPENSSL_PRIVATE STATIC int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx); -STATIC int tor_tls_classify_client_ciphers(const SSL *ssl, +STATIC int tor_tls_classify_client_ciphers(const struct ssl_st *ssl, STACK_OF(SSL_CIPHER) *peer_ciphers); -STATIC int tor_tls_client_is_using_v2_ciphers(const SSL *ssl); +#endif +STATIC int tor_tls_client_is_using_v2_ciphers(const struct ssl_st *ssl); MOCK_DECL(STATIC void, try_to_extract_certs_from_tls, - (int severity, tor_tls_t *tls, X509 **cert_out, X509 **id_cert_out)); + (int severity, tor_tls_t *tls, struct x509_st **cert_out, + struct x509_st **id_cert_out)); #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY -STATIC size_t SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, +STATIC size_t SSL_SESSION_get_master_key(struct ssl_session_st *s, + uint8_t *out, size_t len); #endif -STATIC void tor_tls_debug_state_callback(const SSL *ssl, int type, int val); -STATIC void tor_tls_server_info_callback(const SSL *ssl, int type, int val); -STATIC int tor_tls_session_secret_cb(SSL *ssl, void *secret, +STATIC void tor_tls_debug_state_callback(const struct ssl_st *ssl, + int type, int val); +STATIC void tor_tls_server_info_callback(const struct ssl_st *ssl, + int type, int val); +#ifdef TORTLS_OPENSSL_PRIVATE +STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret, int *secret_len, STACK_OF(SSL_CIPHER) *peer_ciphers, CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher, void *arg); STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher); -MOCK_DECL(STATIC X509*, tor_tls_create_certificate,(crypto_pk_t *rsa, +#endif /* defined(TORTLS_OPENSSL_PRIVATE) */ +MOCK_DECL(STATIC struct x509_st *, tor_tls_create_certificate, + (crypto_pk_t *rsa, crypto_pk_t *rsa_sign, const char *cname, const char *cname_sign, unsigned int cert_lifetime)); STATIC tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, unsigned flags, int is_client); -MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,(X509 *x509_cert)); +MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new, + (struct x509_st *x509_cert)); STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext, crypto_pk_t *identity, unsigned int key_lifetime, @@ -172,10 +187,16 @@ extern tor_tls_context_t *client_tls_context; extern uint16_t v2_cipher_list[]; extern uint64_t total_bytes_written_over_tls; extern uint64_t total_bytes_written_by_tls; -#endif -#endif /* endif TORTLS_PRIVATE */ +STATIC tor_x509_cert_t *tor_x509_cert_replace_expiration( + const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key); +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TORTLS_PRIVATE) */ +tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); const char *tor_tls_err_to_string(int err); void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz); @@ -197,12 +218,12 @@ void tor_tls_set_renegotiate_callback(tor_tls_t *tls, int tor_tls_is_server(tor_tls_t *tls); void tor_tls_free(tor_tls_t *tls); int tor_tls_peer_has_cert(tor_tls_t *tls); -tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); MOCK_DECL(tor_x509_cert_t *,tor_tls_get_peer_cert,(tor_tls_t *tls)); MOCK_DECL(tor_x509_cert_t *,tor_tls_get_own_cert,(tor_tls_t *tls)); int tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity); int tor_tls_check_lifetime(int severity, - tor_tls_t *tls, int past_tolerance, + tor_tls_t *tls, time_t now, + int past_tolerance, int future_tolerance); MOCK_DECL(int, tor_tls_read, (tor_tls_t *tls, char *cp, size_t len)); int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n); @@ -228,6 +249,11 @@ int tor_tls_used_v1_handshake(tor_tls_t *tls); int tor_tls_get_num_server_handshakes(tor_tls_t *tls); int tor_tls_server_got_renegotiate(tor_tls_t *tls); MOCK_DECL(int,tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out)); +MOCK_DECL(int,tor_tls_export_key_material,( + tor_tls_t *tls, uint8_t *secrets_out, + const uint8_t *context, + size_t context_len, + const char *label)); /* Log and abort if there are unhandled TLS errors in OpenSSL's error stack. */ @@ -256,10 +282,11 @@ MOCK_DECL(int,tor_tls_cert_matches_key,(const tor_tls_t *tls, int tor_tls_cert_is_valid(int severity, const tor_x509_cert_t *cert, const tor_x509_cert_t *signing_cert, + time_t now, int check_rsa_1024); const char *tor_tls_get_ciphersuite_name(tor_tls_t *tls); int evaluate_ecgroup_for_tls(const char *ecgroup); -#endif +#endif /* !defined(TOR_TORTLS_H) */ diff --git a/src/common/util.c b/src/common/util.c index d2cbacde31..5ff7e104d6 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -31,11 +31,11 @@ #include <process.h> #include <tchar.h> #include <winbase.h> -#else +#else /* !(defined(_WIN32)) */ #include <dirent.h> #include <pwd.h> #include <grp.h> -#endif +#endif /* defined(_WIN32) */ /* math.h needs this on Linux */ #ifndef _USE_ISOC99_ @@ -79,13 +79,13 @@ #include <malloc/malloc.h> #endif #ifdef HAVE_MALLOC_H -#if !defined(OPENBSD) && !defined(__FreeBSD__) +#if !defined(OpenBSD) && !defined(__FreeBSD__) /* OpenBSD has a malloc.h, but for our purposes, it only exists in order to * scold us for being so stupid as to autodetect its presence. To be fair, * they've done this since 1996, when autoconf was only 5 years old. */ #include <malloc.h> -#endif -#endif +#endif /* !defined(OpenBSD) && !defined(__FreeBSD__) */ +#endif /* defined(HAVE_MALLOC_H) */ #ifdef HAVE_MALLOC_NP_H #include <malloc_np.h> #endif @@ -116,12 +116,12 @@ dmalloc_strndup(file, line, (string), -1, xalloc_b) #else #error "No dmalloc_strdup or equivalent" - #endif +#endif /* defined(HAVE_DMALLOC_STRDUP) || ... */ -#else /* not using dmalloc */ +#else /* !(defined(USE_DMALLOC)) */ #define DMALLOC_FN_ARGS -#endif +#endif /* defined(USE_DMALLOC) */ /** Allocate a chunk of <b>size</b> bytes of memory, and return a pointer to * result. On error, log and terminate the process. (Same as malloc(size), @@ -142,7 +142,7 @@ tor_malloc_(size_t size DMALLOC_PARAMS) if (size==0) { size=1; } -#endif +#endif /* !defined(MALLOC_ZERO_WORKS) */ #ifdef USE_DMALLOC result = dmalloc_malloc(file, line, size, DMALLOC_FUNC_MALLOC, 0, 0); @@ -187,8 +187,9 @@ tor_malloc_zero_(size_t size DMALLOC_PARAMS) * 0xfffe0001. */ #define SQRT_SIZE_MAX_P1 (((size_t)1) << (sizeof(size_t)*4)) -/** Return non-zero if and only if the product of the arguments is exact. */ -static inline int +/** Return non-zero if and only if the product of the arguments is exact, + * and cannot overflow. */ +int size_mul_check(const size_t x, const size_t y) { /* This first check is equivalent to @@ -202,15 +203,6 @@ size_mul_check(const size_t x, const size_t y) x <= SIZE_MAX / y); } -#ifdef TOR_UNIT_TESTS -/** Exposed for unit tests only */ -int -size_mul_check__(const size_t x, const size_t y) -{ - return size_mul_check(x,y); -} -#endif - /** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill * the memory with zero bytes, and return a pointer to the result. * Log and terminate the process on error. (Same as @@ -241,7 +233,7 @@ tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS) if (size==0) { size=1; } -#endif +#endif /* !defined(MALLOC_ZERO_WORKS) */ #ifdef USE_DMALLOC result = dmalloc_realloc(file, line, ptr, size, DMALLOC_FUNC_REALLOC, 0); @@ -370,16 +362,16 @@ tor_log_mallinfo(int severity) mi.arena, mi.ordblks, mi.smblks, mi.hblks, mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks, mi.keepcost); -#else +#else /* !(defined(HAVE_MALLINFO)) */ (void)severity; -#endif +#endif /* defined(HAVE_MALLINFO) */ #ifdef USE_DMALLOC dmalloc_log_changed(0, /* Since the program started. */ 1, /* Log info about non-freed pointers. */ 0, /* Do not log info about freed pointers. */ 0 /* Do not log individual pointers. */ ); -#endif +#endif /* defined(USE_DMALLOC) */ } ENABLE_GCC_WARNING(aggregate-return) @@ -409,7 +401,7 @@ tor_lround(double d) return (long)rint(d); #else return (long)(d > 0 ? d + 0.5 : ceil(d - 0.5)); -#endif +#endif /* defined(HAVE_LROUND) || ... */ } /** Return the 64-bit integer closest to d. We define this wrapper here so @@ -424,7 +416,7 @@ tor_llround(double d) return (int64_t)rint(d); #else return (int64_t)(d > 0 ? d + 0.5 : ceil(d - 0.5)); -#endif +#endif /* defined(HAVE_LLROUND) || ... */ } /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */ @@ -453,7 +445,7 @@ tor_log2(uint64_t u64) r += 2; } if (u64 >= (U64_LITERAL(1)<<1)) { - u64 >>= 1; + // u64 >>= 1; // not using this any more. r += 1; } return r; @@ -485,7 +477,7 @@ round_to_power_of_2(uint64_t u64) /** Return the lowest x such that x is at least <b>number</b>, and x modulo * <b>divisor</b> == 0. If no such x can be expressed as an unsigned, return - * UINT_MAX */ + * UINT_MAX. Asserts if divisor is zero. */ unsigned round_to_next_multiple_of(unsigned number, unsigned divisor) { @@ -499,7 +491,7 @@ round_to_next_multiple_of(unsigned number, unsigned divisor) /** Return the lowest x such that x is at least <b>number</b>, and x modulo * <b>divisor</b> == 0. If no such x can be expressed as a uint32_t, return - * UINT32_MAX */ + * UINT32_MAX. Asserts if divisor is zero. */ uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) { @@ -514,7 +506,7 @@ round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) /** Return the lowest x such that x is at least <b>number</b>, and x modulo * <b>divisor</b> == 0. If no such x can be expressed as a uint64_t, return - * UINT64_MAX */ + * UINT64_MAX. Asserts if divisor is zero. */ uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) { @@ -712,6 +704,19 @@ tor_strisnonupper(const char *s) return 1; } +/** Return true iff every character in <b>s</b> is whitespace space; else + * return false. */ +int +tor_strisspace(const char *s) +{ + while (*s) { + if (!TOR_ISSPACE(*s)) + return 0; + s++; + } + return 1; +} + /** As strcmp, except that either string may be NULL. The NULL string is * considered to be before any non-NULL string. */ int @@ -1167,7 +1172,7 @@ tor_parse_long(const char *s, int base, long min, long max, char *endptr; long r; - if (base < 0) { + if (BUG(base < 0)) { if (ok) *ok = 0; return 0; @@ -1186,7 +1191,7 @@ tor_parse_ulong(const char *s, int base, unsigned long min, char *endptr; unsigned long r; - if (base < 0) { + if (BUG(base < 0)) { if (ok) *ok = 0; return 0; @@ -1218,7 +1223,7 @@ tor_parse_uint64(const char *s, int base, uint64_t min, char *endptr; uint64_t r; - if (base < 0) { + if (BUG(base < 0)) { if (ok) *ok = 0; return 0; @@ -1228,20 +1233,12 @@ tor_parse_uint64(const char *s, int base, uint64_t min, #ifdef HAVE_STRTOULL r = (uint64_t)strtoull(s, &endptr, base); #elif defined(_WIN32) -#if defined(_MSC_VER) && _MSC_VER < 1300 - tor_assert(base <= 10); - r = (uint64_t)_atoi64(s); - endptr = (char*)s; - while (TOR_ISSPACE(*endptr)) endptr++; - while (TOR_ISDIGIT(*endptr)) endptr++; -#else r = (uint64_t)_strtoui64(s, &endptr, base); -#endif #elif SIZEOF_LONG == 8 r = (uint64_t)strtoul(s, &endptr, base); #else #error "I don't know how to parse 64-bit numbers." -#endif +#endif /* defined(HAVE_STRTOULL) || ... */ CHECK_STRTOX_RESULT(); } @@ -1639,7 +1636,7 @@ tor_timegm(const struct tm *tm, time_t *time_out) log_warn(LD_BUG, "Result does not fit in tor_timegm"); return -1; } -#endif +#endif /* SIZEOF_TIME_T < 8 */ *time_out = (time_t)seconds; return 0; } @@ -1803,17 +1800,26 @@ format_iso_time_nospace_usec(char *buf, const struct timeval *tv) /** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>, * parse it and store its value in *<b>t</b>. Return 0 on success, -1 on * failure. Ignore extraneous stuff in <b>cp</b> after the end of the time - * string, unless <b>strict</b> is set. */ + * string, unless <b>strict</b> is set. If <b>nospace</b> is set, + * expect the YYYY-MM-DDTHH:MM:SS format. */ int -parse_iso_time_(const char *cp, time_t *t, int strict) +parse_iso_time_(const char *cp, time_t *t, int strict, int nospace) { struct tm st_tm; unsigned int year=0, month=0, day=0, hour=0, minute=0, second=0; int n_fields; - char extra_char; - n_fields = tor_sscanf(cp, "%u-%2u-%2u %2u:%2u:%2u%c", &year, &month, - &day, &hour, &minute, &second, &extra_char); - if (strict ? (n_fields != 6) : (n_fields < 6)) { + char extra_char, separator_char; + n_fields = tor_sscanf(cp, "%u-%2u-%2u%c%2u:%2u:%2u%c", + &year, &month, &day, + &separator_char, + &hour, &minute, &second, &extra_char); + if (strict ? (n_fields != 7) : (n_fields < 7)) { + char *esc = esc_for_log(cp); + log_warn(LD_GENERAL, "ISO time %s was unparseable", esc); + tor_free(esc); + return -1; + } + if (separator_char != (nospace ? 'T' : ' ')) { char *esc = esc_for_log(cp); log_warn(LD_GENERAL, "ISO time %s was unparseable", esc); tor_free(esc); @@ -1855,7 +1861,16 @@ parse_iso_time_(const char *cp, time_t *t, int strict) int parse_iso_time(const char *cp, time_t *t) { - return parse_iso_time_(cp, t, 1); + return parse_iso_time_(cp, t, 1, 0); +} + +/** + * As parse_iso_time, but parses a time encoded by format_iso_time_nospace(). + */ +int +parse_iso_time_nospace(const char *cp, time_t *t) +{ + return parse_iso_time_(cp, t, 1, 1); } /** Given a <b>date</b> in one of the three formats allowed by HTTP (ugh), @@ -1930,7 +1945,7 @@ parse_http_time(const char *date, struct tm *tm) /** Given an <b>interval</b> in seconds, try to write it to the * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form. - * Return 0 on success, -1 on failure. + * Returns a non-negative integer on success, -1 on failure. */ int format_time_interval(char *out, size_t out_len, long interval) @@ -1999,7 +2014,7 @@ update_approx_time(time_t now) { cached_approx_time = now; } -#endif +#endif /* !defined(TIME_IS_FAST) */ /* ===== * Rate limiting @@ -2095,7 +2110,7 @@ read_all(tor_socket_t fd, char *buf, size_t count, int isSocket) return -1; } - while (numread != count) { + while (numread < count) { if (isSocket) result = tor_socket_recv(fd, buf+numread, count-numread, 0); else @@ -2128,9 +2143,9 @@ clean_name_for_stat(char *name) return; name[len-1]='\0'; } -#else +#else /* !(defined(_WIN32)) */ (void)name; -#endif +#endif /* defined(_WIN32) */ } /** Wrapper for unlink() to make it mockable for the test suite; returns 0 @@ -2270,10 +2285,14 @@ check_private_dir,(const char *dirname, cpd_check_t check, * permissions on the directory will be checked again below.*/ fd = open(sandbox_intern_string(dirname), O_NOFOLLOW); - if (fd == -1) + if (fd == -1) { + log_warn(LD_FS, "Could not reopen recently created directory %s: %s", + dirname, + strerror(errno)); return -1; - else + } else { close(fd); + } } else if (!(check & CPD_CHECK)) { log_warn(LD_FS, "Directory %s does not exist.", dirname); @@ -2323,21 +2342,27 @@ check_private_dir,(const char *dirname, cpd_check_t check, running_gid = getgid(); } if (st.st_uid != running_uid) { - const struct passwd *pw_uid = NULL; - char *process_ownername = NULL; + char *process_ownername = NULL, *file_ownername = NULL; - pw_uid = tor_getpwuid(running_uid); - process_ownername = pw_uid ? tor_strdup(pw_uid->pw_name) : - tor_strdup("<unknown>"); + { + const struct passwd *pw_running = tor_getpwuid(running_uid); + process_ownername = pw_running ? tor_strdup(pw_running->pw_name) : + tor_strdup("<unknown>"); + } - pw_uid = tor_getpwuid(st.st_uid); + { + const struct passwd *pw_stat = tor_getpwuid(st.st_uid); + file_ownername = pw_stat ? tor_strdup(pw_stat->pw_name) : + tor_strdup("<unknown>"); + } log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by " "%s (%d). Perhaps you are running Tor as the wrong user?", - dirname, process_ownername, (int)running_uid, - pw ? pw->pw_name : "<unknown>", (int)st.st_uid); + dirname, process_ownername, (int)running_uid, + file_ownername, (int)st.st_uid); tor_free(process_ownername); + tor_free(file_ownername); close(fd); return -1; } @@ -2394,7 +2419,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, } } close(fd); -#else +#else /* !(!defined(_WIN32)) */ /* Win32 case: we can't open() a directory. */ (void)effective_user; @@ -2428,7 +2453,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, return -1; } -#endif +#endif /* !defined(_WIN32) */ return 0; } @@ -2448,7 +2473,7 @@ write_str_to_file,(const char *fname, const char *str, int bin)) "We're writing a text string that already contains a CR to %s", escaped(fname)); } -#endif +#endif /* defined(_WIN32) */ return write_bytes_to_file(fname, str, strlen(str), bin); } @@ -2601,6 +2626,14 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write) if (file_data->rename_on_close) { tor_assert(file_data->tempname && file_data->filename); + if (!abort_write) { + tor_assert(strcmp(file_data->filename, file_data->tempname)); + if (replace_file(file_data->tempname, file_data->filename)) { + log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename, + strerror(errno)); + abort_write = r = -1; + } + } if (abort_write) { int res = unlink(file_data->tempname); if (res != 0) { @@ -2609,13 +2642,6 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write) file_data->tempname, strerror(errno)); r = -1; } - } else { - tor_assert(strcmp(file_data->filename, file_data->tempname)); - if (replace_file(file_data->tempname, file_data->filename)) { - log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename, - strerror(errno)); - r = -1; - } } } @@ -2847,7 +2873,7 @@ read_file_to_str, (const char *filename, int flags, struct stat *stat_out)) errno = save_errno; return string; } -#endif +#endif /* !defined(_WIN32) */ if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) { close(fd); @@ -2880,7 +2906,7 @@ read_file_to_str, (const char *filename, int flags, struct stat *stat_out)) if (!bin) { statbuf.st_size = (size_t) r; } else -#endif +#endif /* defined(_WIN32) || defined(__CYGWIN__) */ if (r != statbuf.st_size) { /* Unless we're using text mode on win32, we'd better have an exact * match for size. */ @@ -2954,8 +2980,9 @@ unescape_string(const char *s, char **result, size_t *size_out) *out = '\0'; if (size_out) *size_out = out - *result; return cp+1; - case '\0': + /* LCOV_EXCL_START -- we caught this in parse_config_from_line. */ + case '\0': tor_fragile_assert(); tor_free(*result); return NULL; @@ -3003,8 +3030,9 @@ unescape_string(const char *s, char **result, size_t *size_out) *out++ = cp[1]; cp += 2; break; - default: + /* LCOV_EXCL_START */ + default: /* we caught this above in the initial loop. */ tor_assert_nonfatal_unreached(); tor_free(*result); return NULL; @@ -3017,135 +3045,39 @@ unescape_string(const char *s, char **result, size_t *size_out) } } -/** Given a string containing part of a configuration file or similar format, - * advance past comments and whitespace and try to parse a single line. If we - * parse a line successfully, set *<b>key_out</b> to a new string holding the - * key portion and *<b>value_out</b> to a new string holding the value portion - * of the line, and return a pointer to the start of the next line. If we run - * out of data, return a pointer to the end of the string. If we encounter an - * error, return NULL and set *<b>err_out</b> (if provided) to an error - * message. - */ -const char * -parse_config_line_from_str_verbose(const char *line, char **key_out, - char **value_out, - const char **err_out) +/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the + * enclosing quotes. Backslashes are not unescaped. Return the unquoted + * <b>path</b> on sucess or 0 if <b>path</b> is not quoted correctly. */ +char * +get_unquoted_path(const char *path) { - /* - See torrc_format.txt for a description of the (silly) format this parses. - */ - const char *key, *val, *cp; - int continuation = 0; - - tor_assert(key_out); - tor_assert(value_out); + size_t len = strlen(path); - *key_out = *value_out = NULL; - key = val = NULL; - /* Skip until the first keyword. */ - while (1) { - while (TOR_ISSPACE(*line)) - ++line; - if (*line == '#') { - while (*line && *line != '\n') - ++line; - } else { - break; - } + if (len == 0) { + return tor_strdup(""); } - if (!*line) { /* End of string? */ - *key_out = *value_out = NULL; - return line; + int has_start_quote = (path[0] == '\"'); + int has_end_quote = (len > 0 && path[len-1] == '\"'); + if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) { + return NULL; } - /* Skip until the next space or \ followed by newline. */ - key = line; - while (*line && !TOR_ISSPACE(*line) && *line != '#' && - ! (line[0] == '\\' && line[1] == '\n')) - ++line; - *key_out = tor_strndup(key, line-key); - - /* Skip until the value. */ - while (*line == ' ' || *line == '\t') - ++line; - - val = line; - - /* Find the end of the line. */ - if (*line == '\"') { // XXX No continuation handling is done here - if (!(line = unescape_string(line, value_out, NULL))) { - if (err_out) - *err_out = "Invalid escape sequence in quoted string"; - return NULL; - } - while (*line == ' ' || *line == '\t') - ++line; - if (*line == '\r' && *(++line) == '\n') - ++line; - if (*line && *line != '#' && *line != '\n') { - if (err_out) - *err_out = "Excess data after quoted string"; + char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1); + char *s = unquoted_path; + size_t i; + for (i = has_start_quote; i < len - has_end_quote; i++) { + if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) { + *(s-1) = path[i]; + } else if (path[i] != '\"') { + *s++ = path[i]; + } else { /* unescaped quote */ + tor_free(unquoted_path); return NULL; } - } else { - /* Look for the end of the line. */ - while (*line && *line != '\n' && (*line != '#' || continuation)) { - if (*line == '\\' && line[1] == '\n') { - continuation = 1; - line += 2; - } else if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); - if (*line == '\n') - ++line; - } else { - ++line; - } - } - - if (*line == '\n') { - cp = line++; - } else { - cp = line; - } - /* Now back cp up to be the last nonspace character */ - while (cp>val && TOR_ISSPACE(*(cp-1))) - --cp; - - tor_assert(cp >= val); - - /* Now copy out and decode the value. */ - *value_out = tor_strndup(val, cp-val); - if (continuation) { - char *v_out, *v_in; - v_out = v_in = *value_out; - while (*v_in) { - if (*v_in == '#') { - do { - ++v_in; - } while (*v_in && *v_in != '\n'); - if (*v_in == '\n') - ++v_in; - } else if (v_in[0] == '\\' && v_in[1] == '\n') { - v_in += 2; - } else { - *v_out++ = *v_in++; - } - } - *v_out = '\0'; - } } - - if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); - } - while (TOR_ISSPACE(*line)) ++line; - - return line; + *s = '\0'; + return unquoted_path; } /** Expand any homedir prefix on <b>filename</b>; return a newly allocated @@ -3160,7 +3092,7 @@ expand_filename(const char *filename) * Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/ */ return tor_strdup(filename); -#else +#else /* !(defined(_WIN32)) */ if (*filename == '~') { char *home, *result=NULL; const char *rest; @@ -3190,10 +3122,10 @@ expand_filename(const char *filename) } tor_free(username); rest = slash ? (slash+1) : ""; -#else +#else /* !(defined(HAVE_PWD_H)) */ log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h"); return tor_strdup(filename); -#endif +#endif /* defined(HAVE_PWD_H) */ } tor_assert(home); /* Remove trailing slash. */ @@ -3206,7 +3138,7 @@ expand_filename(const char *filename) } else { return tor_strdup(filename); } -#endif +#endif /* defined(_WIN32) */ } #define MAX_SCANF_WIDTH 9999 @@ -3534,6 +3466,17 @@ smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern, smartlist_add(sl, str); } +/** Append a copy of string to sl */ +void +smartlist_add_strdup(struct smartlist_t *sl, const char *string) +{ + char *copy; + + copy = tor_strdup(string); + + smartlist_add(sl, copy); +} + /** Return a new list containing the filenames in the directory <b>dirname</b>. * Return NULL on error or if <b>dirname</b> is not a directory. */ @@ -3564,10 +3507,10 @@ tor_listdir, (const char *dirname)) name[sizeof(name)-1] = '\0'; #else strlcpy(name,findData.cFileName,sizeof(name)); -#endif +#endif /* defined(UNICODE) */ if (strcmp(name, ".") && strcmp(name, "..")) { - smartlist_add(result, tor_strdup(name)); + smartlist_add_strdup(result, name); } if (!FindNextFile(handle, &findData)) { DWORD err; @@ -3581,7 +3524,7 @@ tor_listdir, (const char *dirname)) } FindClose(handle); tor_free(pattern); -#else +#else /* !(defined(_WIN32)) */ const char *prot_dname = sandbox_intern_string(dirname); DIR *d; struct dirent *de; @@ -3593,10 +3536,10 @@ tor_listdir, (const char *dirname)) if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) continue; - smartlist_add(result, tor_strdup(de->d_name)); + smartlist_add_strdup(result, de->d_name); } closedir(d); -#endif +#endif /* defined(_WIN32) */ return result; } @@ -3612,7 +3555,7 @@ path_is_relative(const char *filename) else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) && filename[1] == ':' && filename[2] == '\\') return 0; -#endif +#endif /* defined(_WIN32) */ else return 1; } @@ -3675,7 +3618,7 @@ start_daemon(void) } else { /* Child */ close(daemon_filedes[0]); /* we only write */ - pid = setsid(); /* Detach from controlling terminal */ + (void) setsid(); /* Detach from controlling terminal */ /* * Fork one more time, so the parent (the session group leader) can exit. * This means that we, as a non-session group leader, can never regain a @@ -3742,7 +3685,7 @@ finish_daemon(const char *desired_cwd) } close(daemon_filedes[1]); } -#else +#else /* !(!defined(_WIN32)) */ /* defined(_WIN32) */ void start_daemon(void) @@ -3753,11 +3696,12 @@ finish_daemon(const char *cp) { (void)cp; } -#endif +#endif /* !defined(_WIN32) */ /** Write the current process ID, followed by NL, into <b>filename</b>. + * Return 0 on success, -1 on failure. */ -void +int write_pidfile(const char *filename) { FILE *pidfile; @@ -3765,13 +3709,19 @@ write_pidfile(const char *filename) if ((pidfile = fopen(filename, "w")) == NULL) { log_warn(LD_FS, "Unable to open \"%s\" for writing: %s", filename, strerror(errno)); + return -1; } else { #ifdef _WIN32 - fprintf(pidfile, "%d\n", (int)_getpid()); + int pid = (int)_getpid(); #else - fprintf(pidfile, "%d\n", (int)getpid()); + int pid = (int)getpid(); #endif - fclose(pidfile); + int rv = 0; + if (fprintf(pidfile, "%d\n", pid) < 0) + rv = -1; + if (fclose(pidfile) < 0) + rv = -1; + return rv; } } @@ -3788,7 +3738,7 @@ load_windows_system_library(const TCHAR *library_name) _tcscat(path, library_name); return LoadLibrary(path); } -#endif +#endif /* defined(_WIN32) */ /** Format a single argument for being put on a Windows command line. * Returns a newly allocated string */ @@ -4083,7 +4033,7 @@ format_helper_exit_status(unsigned char child_state, int saved_errno, done: return res; } -#endif +#endif /* !defined(_WIN32) */ /* Maximum number of file descriptors, if we cannot get it via sysconf() */ #define DEFAULT_MAX_FD 256 @@ -4107,12 +4057,12 @@ tor_terminate_process(process_handle_t *process_handle) else return 0; } -#else /* Unix */ +#else /* !(defined(_WIN32)) */ if (process_handle->waitpid_cb) { /* We haven't got a waitpid yet, so we can just kill off the process. */ return kill(process_handle->pid, SIGTERM); } -#endif +#endif /* defined(_WIN32) */ return 0; /* We didn't need to kill the process, so report success */ } @@ -4134,14 +4084,14 @@ tor_process_get_stdout_pipe(process_handle_t *process_handle) { return process_handle->stdout_pipe; } -#else +#else /* !(defined(_WIN32)) */ /* DOCDOC tor_process_get_stdout_pipe */ -FILE * +int tor_process_get_stdout_pipe(process_handle_t *process_handle) { - return process_handle->stdout_handle; + return process_handle->stdout_pipe; } -#endif +#endif /* defined(_WIN32) */ /* DOCDOC process_handle_new */ static process_handle_t * @@ -4157,7 +4107,7 @@ process_handle_new(void) out->stdin_pipe = -1; out->stdout_pipe = -1; out->stderr_pipe = -1; -#endif +#endif /* defined(_WIN32) */ return out; } @@ -4177,7 +4127,7 @@ process_handle_waitpid_cb(int status, void *arg) process_handle->status = PROCESS_STATUS_NOTRUNNING; process_handle->waitpid_cb = 0; } -#endif +#endif /* !defined(_WIN32) */ /** * @name child-process states @@ -4199,6 +4149,20 @@ process_handle_waitpid_cb(int status, void *arg) #define CHILD_STATE_EXEC 8 #define CHILD_STATE_FAILEXEC 9 /** @} */ +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} /** Start a program in the background. If <b>filename</b> contains a '/', then * it will be treated as an absolute or relative path. Otherwise, on * non-Windows systems, the system path will be searched for <b>filename</b>. @@ -4223,6 +4187,12 @@ tor_spawn_background(const char *const filename, const char **argv, process_environment_t *env, process_handle_t **process_handle_out) { + if (BUG(may_spawn_background_process == 0)) { + /* We should never reach this point if we're forbidden to spawn + * processes. Instead we should have caught the attempt earlier. */ + return PROCESS_STATUS_ERROR; + } + #ifdef _WIN32 HANDLE stdout_pipe_read = NULL; HANDLE stdout_pipe_write = NULL; @@ -4340,13 +4310,12 @@ tor_spawn_background(const char *const filename, const char **argv, /* TODO: Close pipes on exit */ *process_handle_out = process_handle; return status; -#else // _WIN32 +#else /* !(defined(_WIN32)) */ pid_t pid; int stdout_pipe[2]; int stderr_pipe[2]; int stdin_pipe[2]; int fd, retval; - ssize_t nbytes; process_handle_t *process_handle; int status; @@ -4367,7 +4336,7 @@ tor_spawn_background(const char *const filename, const char **argv, and we are not allowed to use unsafe functions between fork and exec */ error_message_length = strlen(error_message); - child_state = CHILD_STATE_PIPE; + // child_state = CHILD_STATE_PIPE; /* Set up pipe for redirecting stdout, stderr, and stdin of child */ retval = pipe(stdout_pipe); @@ -4404,7 +4373,7 @@ tor_spawn_background(const char *const filename, const char **argv, return status; } - child_state = CHILD_STATE_MAXFD; + // child_state = CHILD_STATE_MAXFD; #ifdef _SC_OPEN_MAX if (-1 == max_fd) { @@ -4415,11 +4384,11 @@ tor_spawn_background(const char *const filename, const char **argv, "Cannot find maximum file descriptor, assuming %d", max_fd); } } -#else +#else /* !(defined(_SC_OPEN_MAX)) */ max_fd = DEFAULT_MAX_FD; -#endif +#endif /* defined(_SC_OPEN_MAX) */ - child_state = CHILD_STATE_FORK; + // child_state = CHILD_STATE_FORK; pid = fork(); if (0 == pid) { @@ -4432,7 +4401,7 @@ tor_spawn_background(const char *const filename, const char **argv, * than nothing. */ prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ child_state = CHILD_STATE_DUPOUT; @@ -4455,7 +4424,7 @@ tor_spawn_background(const char *const filename, const char **argv, if (-1 == retval) goto error; - child_state = CHILD_STATE_CLOSEFD; + // child_state = CHILD_STATE_CLOSEFD; close(stderr_pipe[0]); close(stderr_pipe[1]); @@ -4471,7 +4440,7 @@ tor_spawn_background(const char *const filename, const char **argv, close(fd); } - child_state = CHILD_STATE_EXEC; + // child_state = CHILD_STATE_EXEC; /* Call the requested program. We need the cast because execvp doesn't define argv as const, even though it @@ -4490,7 +4459,8 @@ tor_spawn_background(const char *const filename, const char **argv, error: { /* XXX: are we leaking fds from the pipe? */ - int n; + int n, err=0; + ssize_t nbytes; n = format_helper_exit_status(child_state, errno, hex_errno); @@ -4499,13 +4469,14 @@ tor_spawn_background(const char *const filename, const char **argv, value, but there is nothing we can do if it fails */ /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ nbytes = write(STDOUT_FILENO, error_message, error_message_length); + err = (nbytes < 0); nbytes = write(STDOUT_FILENO, hex_errno, n); + err += (nbytes < 0); } - } - (void) nbytes; + _exit(err?254:255); + } - _exit(255); /* Never reached, but avoids compiler warning */ return status; // LCOV_EXCL_LINE } @@ -4570,14 +4541,10 @@ tor_spawn_background(const char *const filename, const char **argv, log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " "nonblocking in parent process: %s", strerror(errno)); } - /* Open the buffered IO streams */ - process_handle->stdout_handle = fdopen(process_handle->stdout_pipe, "r"); - process_handle->stderr_handle = fdopen(process_handle->stderr_pipe, "r"); - process_handle->stdin_handle = fdopen(process_handle->stdin_pipe, "r"); *process_handle_out = process_handle; - return process_handle->status; -#endif // _WIN32 + return status; +#endif /* defined(_WIN32) */ } /** Destroy all resources allocated by the process handle in @@ -4619,18 +4586,13 @@ tor_process_handle_destroy,(process_handle_t *process_handle, if (process_handle->stdin_pipe) CloseHandle(process_handle->stdin_pipe); -#else - if (process_handle->stdout_handle) - fclose(process_handle->stdout_handle); - - if (process_handle->stderr_handle) - fclose(process_handle->stderr_handle); - - if (process_handle->stdin_handle) - fclose(process_handle->stdin_handle); +#else /* !(defined(_WIN32)) */ + close(process_handle->stdout_pipe); + close(process_handle->stderr_pipe); + close(process_handle->stdin_pipe); clear_waitpid_callback(process_handle->waitpid_cb); -#endif +#endif /* defined(_WIN32) */ memset(process_handle, 0x0f, sizeof(process_handle_t)); tor_free(process_handle); @@ -4683,7 +4645,7 @@ tor_get_exit_code(process_handle_t *process_handle, return PROCESS_EXIT_ERROR; } } -#else +#else /* !(defined(_WIN32)) */ int stat_loc; int retval; @@ -4718,7 +4680,7 @@ tor_get_exit_code(process_handle_t *process_handle, if (exit_code != NULL) *exit_code = WEXITSTATUS(stat_loc); -#endif // _WIN32 +#endif /* defined(_WIN32) */ return PROCESS_EXIT_EXITED; } @@ -4864,7 +4826,7 @@ get_current_process_environment_variables(void) char **environ_tmp; /* Not const char ** ? Really? */ for (environ_tmp = get_environment(); *environ_tmp; ++environ_tmp) { - smartlist_add(sl, tor_strdup(*environ_tmp)); + smartlist_add_strdup(sl, *environ_tmp); } return sl; @@ -4913,7 +4875,7 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { + while (numread < count) { /* Check if there is anything to read */ retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); if (!retval) { @@ -4958,20 +4920,20 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, } return (ssize_t)numread; } -#else -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If +#else /* !(defined(_WIN32)) */ +/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If * <b>process</b> is NULL, the function will return immediately if there is * nothing more to read. Otherwise data will be read until end of file, or * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the * file has been reached. */ ssize_t -tor_read_all_handle(FILE *h, char *buf, size_t count, +tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof) { size_t numread = 0; - char *retval; + ssize_t result; if (eof) *eof = 0; @@ -4979,37 +4941,31 @@ tor_read_all_handle(FILE *h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { - /* Use fgets because that is what we use in log_from_pipe() */ - retval = fgets(buf+numread, (int)(count-numread), h); - if (NULL == retval) { - if (feof(h)) { - log_debug(LD_GENERAL, "fgets() reached end of file"); - if (eof) - *eof = 1; + while (numread < count) { + result = read(fd, buf+numread, count-numread); + + if (result == 0) { + log_debug(LD_GENERAL, "read() reached end of file"); + if (eof) + *eof = 1; + break; + } else if (result < 0 && errno == EAGAIN) { + if (process) + continue; + else break; - } else { - if (EAGAIN == errno) { - if (process) - continue; - else - break; - } else { - log_warn(LD_GENERAL, "fgets() from handle failed: %s", - strerror(errno)); - return -1; - } - } + } else if (result < 0) { + log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); + return -1; } - tor_assert(retval != NULL); - tor_assert(strlen(retval) + numread <= count); - numread += strlen(retval); + + numread += result; } - log_debug(LD_GENERAL, "fgets() read %d bytes from handle", (int)numread); + log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); return (ssize_t)numread; } -#endif +#endif /* defined(_WIN32) */ /** Read from stdout of a process until the process exits. */ ssize_t @@ -5020,9 +4976,9 @@ tor_read_all_from_process_stdout(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stdout_handle, buf, count, + return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle, NULL); -#endif +#endif /* defined(_WIN32) */ } /** Read from stdout of a process until the process exits. */ @@ -5034,9 +4990,9 @@ tor_read_all_from_process_stderr(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stderr_handle, buf, count, + return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle, NULL); -#endif +#endif /* defined(_WIN32) */ } /** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be @@ -5225,14 +5181,13 @@ log_from_handle(HANDLE *pipe, int severity) return 0; } -#else +#else /* !(defined(_WIN32)) */ /** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set + * <b>fd</b>. Return NULL on error, and set * <b>stream_status_out</b> appropriately. */ MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (FILE *handle, - enum stream_status *stream_status_out)) +tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) { enum stream_status stream_status; char stdout_buf[400]; @@ -5241,13 +5196,13 @@ tor_get_lines_from_handle, (FILE *handle, while (1) { memset(stdout_buf, 0, sizeof(stdout_buf)); - stream_status = get_string_from_pipe(handle, + stream_status = get_string_from_pipe(fd, stdout_buf, sizeof(stdout_buf) - 1); if (stream_status != IO_STREAM_OKAY) goto done; if (!lines) lines = smartlist_new(); - smartlist_add(lines, tor_strdup(stdout_buf)); + smartlist_split_string(lines, stdout_buf, "\n", 0, 0); } done: @@ -5255,20 +5210,20 @@ tor_get_lines_from_handle, (FILE *handle, return lines; } -/** Read from stream, and send lines to log at the specified log level. +/** Read from fd, and send lines to log at the specified log level. * Returns 1 if stream is closed normally, -1 if there is a error reading, and * 0 otherwise. Handles lines from tor-fw-helper and * tor_spawn_background() specially. */ static int -log_from_pipe(FILE *stream, int severity, const char *executable, +log_from_pipe(int fd, int severity, const char *executable, int *child_status) { char buf[256]; enum stream_status r; for (;;) { - r = get_string_from_pipe(stream, buf, sizeof(buf) - 1); + r = get_string_from_pipe(fd, buf, sizeof(buf) - 1); if (r == IO_STREAM_CLOSED) { return 1; @@ -5291,9 +5246,9 @@ log_from_pipe(FILE *stream, int severity, const char *executable, /* We should never get here */ return -1; } -#endif +#endif /* defined(_WIN32) */ -/** Reads from <b>stream</b> and stores input in <b>buf_out</b> making +/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making * sure it's below <b>count</b> bytes. * If the string has a trailing newline, we strip it off. * @@ -5309,52 +5264,28 @@ log_from_pipe(FILE *stream, int severity, const char *executable, * IO_STREAM_OKAY: If everything went okay and we got a string * in <b>buf_out</b>. */ enum stream_status -get_string_from_pipe(FILE *stream, char *buf_out, size_t count) +get_string_from_pipe(int fd, char *buf_out, size_t count) { - char *retval; - size_t len; + ssize_t ret; tor_assert(count <= INT_MAX); - retval = fgets(buf_out, (int)count, stream); + ret = read(fd, buf_out, count); - if (!retval) { - if (feof(stream)) { - /* Program has closed stream (probably it exited) */ - /* TODO: check error */ - return IO_STREAM_CLOSED; - } else { - if (EAGAIN == errno) { - /* Nothing more to read, try again next time */ - return IO_STREAM_EAGAIN; - } else { - /* There was a problem, abandon this child process */ - return IO_STREAM_TERM; - } - } - } else { - len = strlen(buf_out); - if (len == 0) { - /* this probably means we got a NUL at the start of the string. */ - return IO_STREAM_EAGAIN; - } + if (ret == 0) + return IO_STREAM_CLOSED; + else if (ret < 0 && errno == EAGAIN) + return IO_STREAM_EAGAIN; + else if (ret < 0) + return IO_STREAM_TERM; - if (buf_out[len - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[len - 1] = '\0'; - } else { - /* No newline; check whether we overflowed the buffer */ - if (!feof(stream)) - log_info(LD_GENERAL, - "Line from stream was truncated: %s", buf_out); - /* TODO: What to do with this error? */ - } - - return IO_STREAM_OKAY; - } + if (buf_out[ret - 1] == '\n') { + /* Remove the trailing newline */ + buf_out[ret - 1] = '\0'; + } else + buf_out[ret] = '\0'; - /* We should never get here */ - return IO_STREAM_TERM; + return IO_STREAM_OKAY; } /** Parse a <b>line</b> from tor-fw-helper and issue an appropriate @@ -5567,7 +5498,7 @@ tor_check_port_forwarding(const char *filename, status = tor_spawn_background(NULL, argv, NULL, &child_handle); #else status = tor_spawn_background(filename, argv, NULL, &child_handle); -#endif +#endif /* defined(_WIN32) */ tor_free_((void*)argv); argv=NULL; @@ -5591,9 +5522,9 @@ tor_check_port_forwarding(const char *filename, #ifdef _WIN32 stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO); #else - stderr_status = log_from_pipe(child_handle->stderr_handle, + stderr_status = log_from_pipe(child_handle->stderr_pipe, LOG_INFO, filename, &retval); -#endif +#endif /* defined(_WIN32) */ if (handle_fw_helper_output(filename, child_handle) < 0) { log_warn(LD_GENERAL, "Failed to handle fw helper output."); stdout_status = -1; @@ -5618,13 +5549,13 @@ tor_check_port_forwarding(const char *filename, * between log_from_handle and tor_get_exit_code? */ retval = 1; } -#else +#else /* !(defined(_WIN32)) */ else if (1 == stdout_status || 1 == stderr_status) /* stdout or stderr was closed, the process probably * exited. It will be reaped by waitpid() in main.c */ /* TODO: Do something with the process return value */ retval = 1; -#endif +#endif /* defined(_WIN32) */ else /* Both are fine */ retval = 0; @@ -5695,7 +5626,7 @@ clamp_double_to_int64(double number) { int exponent; -#if (defined(__MINGW32__) || defined(__MINGW64__)) && GCC_VERSION >= 409 +#if defined(MINGW_ANY) && GCC_VERSION >= 409 /* Mingw's math.h uses gcc's __builtin_choose_expr() facility to declare isnan, isfinite, and signbit. But as implemented in at least some @@ -5704,7 +5635,7 @@ clamp_double_to_int64(double number) */ #define PROBLEMATIC_FLOAT_CONVERSION_WARNING DISABLE_GCC_WARNING(float-conversion) -#endif +#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */ /* With clang 4.0 we apparently run into "double promotion" warnings here, @@ -5715,7 +5646,7 @@ DISABLE_GCC_WARNING(float-conversion) #define PROBLEMATIC_DOUBLE_PROMOTION_WARNING DISABLE_GCC_WARNING(double-promotion) #endif -#endif +#endif /* defined(__clang__) */ /* NaN is a special case that can't be used with the logic below. */ if (isnan(number)) { @@ -5762,7 +5693,7 @@ tor_htonll(uint64_t a) /* Little endian. The worst... */ return htonl((uint32_t)(a>>32)) | (((uint64_t)htonl((uint32_t)a))<<32); -#endif /* WORDS_BIGENDIAN */ +#endif /* defined(WORDS_BIGENDIAN) */ } /** Return a uint64_t value from <b>a</b> in host byte order. */ diff --git a/src/common/util.h b/src/common/util.h index 479fc8d610..6bc853da26 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -45,7 +45,7 @@ #else #define DMALLOC_PARAMS #define DMALLOC_ARGS -#endif +#endif /* defined(USE_DMALLOC) */ /* Memory management */ void *tor_malloc_(size_t size DMALLOC_PARAMS) ATTR_MALLOC; @@ -72,7 +72,7 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, (p)=NULL; \ } \ STMT_END -#else +#else /* !(defined(USE_DMALLOC)) */ /** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, etc. * Unlike the free() function, tor_free() will still work on NULL pointers, * and it sets the pointer value to NULL after freeing it. @@ -86,7 +86,7 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, (p)=NULL; \ } \ STMT_END -#endif +#endif /* defined(USE_DMALLOC) */ #define tor_malloc(size) tor_malloc_(size DMALLOC_ARGS) #define tor_malloc_zero(size) tor_malloc_zero_(size DMALLOC_ARGS) @@ -109,19 +109,11 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, void tor_log_mallinfo(int severity); -/** Return the offset of <b>member</b> within the type <b>tp</b>, in bytes */ -#if defined(__GNUC__) && __GNUC__ > 3 -#define STRUCT_OFFSET(tp, member) __builtin_offsetof(tp, member) -#else - #define STRUCT_OFFSET(tp, member) \ - ((off_t) (((char*)&((tp*)0)->member)-(char*)0)) -#endif - /** Macro: yield a pointer to the field at position <b>off</b> within the * structure <b>st</b>. Example: * <pre> * struct a { int foo; int bar; } x; - * off_t bar_offset = STRUCT_OFFSET(struct a, bar); + * off_t bar_offset = offsetof(struct a, bar); * int *bar_p = STRUCT_VAR_P(&x, bar_offset); * *bar_p = 3; * </pre> @@ -138,7 +130,7 @@ void tor_log_mallinfo(int severity); * </pre> */ #define SUBTYPE_P(p, subtype, basemember) \ - ((void*) ( ((char*)(p)) - STRUCT_OFFSET(subtype, basemember) )) + ((void*) ( ((char*)(p)) - offsetof(subtype, basemember) )) /* Logic */ /** Macro: true if two values have the same boolean value. */ @@ -163,9 +155,9 @@ int64_t clamp_double_to_int64(double number); void simplify_fraction64(uint64_t *numer, uint64_t *denom); /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> - * and positive <b>b</b>. Works on integer types only. Not defined if a+b can - * overflow. */ -#define CEIL_DIV(a,b) (((a)+(b)-1)/(b)) + * and positive <b>b</b>. Works on integer types only. Not defined if a+(b-1) + * can overflow. */ +#define CEIL_DIV(a,b) (((a)+((b)-1))/(b)) /* Return <b>v</b> if it's between <b>min</b> and <b>max</b>. Otherwise * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if @@ -186,6 +178,7 @@ void tor_strlower(char *s) ATTR_NONNULL((1)); void tor_strupper(char *s) ATTR_NONNULL((1)); int tor_strisprint(const char *s) ATTR_NONNULL((1)); int tor_strisnonupper(const char *s) ATTR_NONNULL((1)); +int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2)); int strcmp_len(const char *s1, const char *s2, size_t len) ATTR_NONNULL((1,2)); @@ -239,6 +232,7 @@ void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...) void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern, va_list args) CHECK_PRINTF(2, 0); +void smartlist_add_strdup(struct smartlist_t *sl, const char *string); /* Time helpers */ long tv_udiff(const struct timeval *start, const struct timeval *end); @@ -254,8 +248,9 @@ void format_local_iso_time(char *buf, time_t t); void format_iso_time(char *buf, time_t t); void format_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace_usec(char *buf, const struct timeval *tv); -int parse_iso_time_(const char *cp, time_t *t, int strict); +int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace); int parse_iso_time(const char *buf, time_t *t); +int parse_iso_time_nospace(const char *cp, time_t *t); int parse_http_time(const char *buf, struct tm *tm); int format_time_interval(char *out, size_t out_len, long interval); @@ -266,7 +261,7 @@ int format_time_interval(char *out, size_t out_len, long interval); #else time_t approx_time(void); void update_approx_time(time_t now); -#endif +#endif /* defined(TIME_IS_FAST) */ /* Rate-limiter */ @@ -319,7 +314,7 @@ enum stream_status { const char *stream_status_to_string(enum stream_status stream_status); -enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count); +enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); MOCK_DECL(int,tor_unlink,(const char *pathname)); @@ -386,9 +381,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out) ATTR_MALLOC; const char *unescape_string(const char *s, char **result, size_t *size_out); -const char *parse_config_line_from_str_verbose(const char *line, - char **key_out, char **value_out, - const char **err_out); +char *get_unquoted_path(const char *path); char *expand_filename(const char *filename); MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); int path_is_relative(const char *filename); @@ -396,13 +389,15 @@ int path_is_relative(const char *filename); /* Process helpers */ void start_daemon(void); void finish_daemon(const char *desired_cwd); -void write_pidfile(const char *filename); +int write_pidfile(const char *filename); /* Port forwarding */ void tor_check_port_forwarding(const char *filename, struct smartlist_t *ports_to_forward, time_t now); +void tor_disable_spawning_background_processes(void); + typedef struct process_handle_t process_handle_t; typedef struct process_environment_t process_environment_t; int tor_spawn_background(const char *const filename, const char **argv, @@ -456,13 +451,10 @@ struct process_handle_t { HANDLE stdout_pipe; HANDLE stderr_pipe; PROCESS_INFORMATION pid; -#else +#else /* !(defined(_WIN32)) */ int stdin_pipe; int stdout_pipe; int stderr_pipe; - FILE *stdin_handle; - FILE *stdout_handle; - FILE *stderr_handle; pid_t pid; /** If the process has not given us a SIGCHLD yet, this has the * waitpid_callback_t that gets invoked once it has. Otherwise this @@ -470,9 +462,9 @@ struct process_handle_t { struct waitpid_callback_t *waitpid_cb; /** The exit status reported by waitpid. */ int waitpid_exit_status; -#endif // _WIN32 +#endif /* defined(_WIN32) */ }; -#endif +#endif /* defined(UTIL_PRIVATE) */ /* Return values of tor_get_exit_code() */ #define PROCESS_EXIT_RUNNING 1 @@ -485,10 +477,10 @@ int tor_split_lines(struct smartlist_t *sl, char *buf, int len); ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, const process_handle_t *process); #else -ssize_t tor_read_all_handle(FILE *h, char *buf, size_t count, +ssize_t tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof); -#endif +#endif /* defined(_WIN32) */ ssize_t tor_read_all_from_process_stdout( const process_handle_t *process_handle, char *buf, size_t count); ssize_t tor_read_all_from_process_stderr( @@ -499,7 +491,7 @@ int tor_process_get_pid(process_handle_t *process_handle); #ifdef _WIN32 HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); #else -FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle); +int tor_process_get_stdout_pipe(process_handle_t *process_handle); #endif #ifdef _WIN32 @@ -508,9 +500,9 @@ tor_get_lines_from_handle,(HANDLE *handle, enum stream_status *stream_status)); #else MOCK_DECL(struct smartlist_t *, -tor_get_lines_from_handle,(FILE *handle, +tor_get_lines_from_handle,(int fd, enum stream_status *stream_status)); -#endif +#endif /* defined(_WIN32) */ int tor_terminate_process(process_handle_t *process_handle); @@ -547,15 +539,13 @@ STATIC int format_helper_exit_status(unsigned char child_state, leading minus) and newline (no null) */ #define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ 1 + sizeof(int) * 2 + 1) -#endif +#endif /* !defined(_WIN32) */ -#endif +#endif /* defined(UTIL_PRIVATE) */ -#ifdef TOR_UNIT_TESTS -int size_mul_check__(const size_t x, const size_t y); -#endif +int size_mul_check(const size_t x, const size_t y); #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) -#endif +#endif /* !defined(TOR_UTIL_H) */ diff --git a/src/common/util_bug.c b/src/common/util_bug.c index 08aba47974..126e843866 100644 --- a/src/common/util_bug.c +++ b/src/common/util_bug.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,10 @@ #include "backtrace.h" #include "container.h" +#ifdef __COVERITY__ +int bug_macro_deadcode_dummy__ = 0; +#endif + #ifdef TOR_UNIT_TESTS static void (*failed_assertion_cb)(void) = NULL; static int n_bugs_to_capture = 0; @@ -44,7 +48,7 @@ static void add_captured_bug(const char *s) { --n_bugs_to_capture; - smartlist_add(bug_messages, tor_strdup(s)); + smartlist_add_strdup(bug_messages, s); } /** Set a callback to be invoked when we get any tor_bug_occurred_ * invocation. We use this in the unit tests so that a nonfatal @@ -55,10 +59,10 @@ tor_set_failed_assertion_callback(void (*fn)(void)) { failed_assertion_cb = fn; } -#else +#else /* !(defined(TOR_UNIT_TESTS)) */ #define capturing_bugs() (0) #define add_captured_bug(s) do { } while (0) -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Helper for tor_assert: report the assertion failure. */ void diff --git a/src/common/util_bug.h b/src/common/util_bug.h index 0695806911..be549fde07 100644 --- a/src/common/util_bug.h +++ b/src/common/util_bug.h @@ -1,10 +1,36 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file util_bug.h + * + * \brief Macros to manage assertions, fatal and non-fatal. + * + * Guidelines: All the different kinds of assertion in this file are for + * bug-checking only. Don't write code that can assert based on bad inputs. + * + * We provide two kinds of assertion here: "fatal" and "nonfatal". Use + * nonfatal assertions for any bug you can reasonably recover from -- and + * please, try to recover! Many severe bugs in Tor have been caused by using + * a regular assertion when a nonfatal assertion would have been better. + * + * If you need to check a condition with a nonfatal assertion, AND recover + * from that same condition, consider using the BUG() macro inside a + * conditional. For example: + * + * <code> + * // wrong -- use tor_assert_nonfatal() if you just want an assertion. + * BUG(ptr == NULL); + * + * // okay, but needlessly verbose + * tor_assert_nonfatal(ptr != NULL); + * if (ptr == NULL) { ... } + * + * // this is how we do it: + * if (BUG(ptr == NULL)) { ... } + * </code> **/ #ifndef TOR_UTIL_BUG_H @@ -27,7 +53,7 @@ * security-critical properties. */ #error "Sorry; we don't support building with NDEBUG." -#endif +#endif /* defined(NDEBUG) */ /* Sometimes we don't want to use assertions during branch coverage tests; it * leads to tons of unreached branches which in reality are only assertions we @@ -44,7 +70,7 @@ tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ abort(); \ } STMT_END -#endif +#endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */ #define tor_assert_unreached() tor_assert(0) @@ -59,11 +85,14 @@ */ #ifdef __COVERITY__ +extern int bug_macro_deadcode_dummy__; #undef BUG // Coverity defines this in global headers; let's override it. This is a // magic coverity-only preprocessor thing. -#nodef BUG(x) ((x)?(__coverity_panic__(),1):0) -#endif +// We use this "deadcode_dummy__" trick to prevent coverity from +// complaining about unreachable bug cases. +#nodef BUG(x) ((x)?(__coverity_panic__(),1):(0+bug_macro_deadcode_dummy__)) +#endif /* defined(__COVERITY__) */ #if defined(__COVERITY__) || defined(__clang_analyzer__) // We're running with a static analysis tool: let's treat even nonfatal @@ -114,7 +143,7 @@ (PREDICT_UNLIKELY(cond) ? \ (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \ : 0) -#endif +#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */ #ifdef __GNUC__ #define IF_BUG_ONCE__(cond,var) \ @@ -127,7 +156,7 @@ "!("#cond")", 1); \ } \ PREDICT_UNLIKELY(bool_result); } )) -#else +#else /* !(defined(__GNUC__)) */ #define IF_BUG_ONCE__(cond,var) \ static int var = 0; \ if (PREDICT_UNLIKELY(cond) ? \ @@ -137,7 +166,7 @@ "!("#cond")", 1), \ 1)) \ : 0) -#endif +#endif /* defined(__GNUC__) */ #define IF_BUG_ONCE_VARNAME_(a) \ warning_logged_on_ ## a ## __ #define IF_BUG_ONCE_VARNAME__(a) \ @@ -167,7 +196,7 @@ void tor_capture_bugs_(int n); void tor_end_capture_bugs_(void); const struct smartlist_t *tor_get_captured_bug_log_(void); void tor_set_failed_assertion_callback(void (*fn)(void)); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ -#endif +#endif /* !defined(TOR_UTIL_BUG_H) */ diff --git a/src/common/util_format.c b/src/common/util_format.c index aef9db85c8..e51757a4e8 100644 --- a/src/common/util_format.c +++ b/src/common/util_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,13 +22,16 @@ #include <stdlib.h> /* Return the base32 encoded size in bytes using the source length srclen. - * The NUL terminated byte is added as well since every base32 encoding - * requires enough space for it. */ + * + * (WATCH OUT: This API counts the terminating NUL byte, but + * base64_encode_size does not.) + */ size_t base32_encoded_size(size_t srclen) { size_t enclen; - enclen = CEIL_DIV(srclen*8, 5) + 1; + tor_assert(srclen < SIZE_T_CEILING / 8); + enclen = BASE32_NOPAD_BUFSIZE(srclen); tor_assert(enclen < INT_MAX && enclen > srclen); return enclen; } @@ -41,7 +44,6 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) size_t nbits = srclen * 8; size_t bit; - tor_assert(srclen < SIZE_T_CEILING/8); /* We need enough space for the encoded data and the extra NUL byte. */ tor_assert(base32_encoded_size(srclen) <= destlen); tor_assert(destlen < SIZE_T_CEILING); @@ -51,9 +53,10 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) for (i=0,bit=0; bit < nbits; ++i, bit+=5) { /* set v to the 16-bit value starting at src[bits/8], 0-padded. */ - v = ((uint8_t)src[bit/8]) << 8; - if (bit+5<nbits) - v += (uint8_t)src[(bit/8)+1]; + size_t idx = bit / 8; + v = ((uint8_t)src[idx]) << 8; + if (idx+1 < srclen) + v += (uint8_t)src[idx+1]; /* set u to the 5-bit value at the bit'th bit of buf. */ u = (v >> (11-(bit%8))) & 0x1F; dest[i] = BASE32_CHARS[u]; @@ -133,6 +136,9 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) /** Return the Base64 encoded size of <b>srclen</b> bytes of data in * bytes. * + * (WATCH OUT: This API <em>does not</em> count the terminating NUL byte, + * but base32_encoded_size does.) + * * If <b>flags</b>&BASE64_ENCODE_MULTILINE is true, return the size * of the encoded output as multiline output (64 character, `\n' terminated * lines). @@ -141,19 +147,16 @@ size_t base64_encode_size(size_t srclen, int flags) { size_t enclen; + + /* Use INT_MAX for overflow checking because base64_encode() returns int. */ tor_assert(srclen < INT_MAX); + tor_assert(CEIL_DIV(srclen, 3) < INT_MAX / 4); - if (srclen == 0) - return 0; + enclen = BASE64_LEN(srclen); + if (flags & BASE64_ENCODE_MULTILINE) + enclen += CEIL_DIV(enclen, BASE64_OPENSSL_LINELEN); - enclen = ((srclen - 1) / 3) * 4 + 4; - if (flags & BASE64_ENCODE_MULTILINE) { - size_t remainder = enclen % BASE64_OPENSSL_LINELEN; - enclen += enclen / BASE64_OPENSSL_LINELEN; - if (remainder) - enclen++; - } - tor_assert(enclen < INT_MAX && enclen > srclen); + tor_assert(enclen < INT_MAX && (enclen == 0 || enclen > srclen)); return enclen; } @@ -263,10 +266,13 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, ENCODE_N(3); ENCODE_PAD(); break; + // LCOV_EXCL_START -- we can't reach this point, because we enforce + // 0 <= ncov_idx < 3 in the loop above. default: /* Something went catastrophically wrong. */ - tor_fragile_assert(); // LCOV_EXCL_LINE + tor_fragile_assert(); return -1; + // LCOV_EXCL_STOP } #undef ENCODE_N @@ -310,39 +316,6 @@ base64_encode_nopad(char *dest, size_t destlen, return (int)(out - dest); } -/** As base64_decode, but do not require any padding on the input */ -int -base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen) -{ - if (srclen > SIZE_T_CEILING - 4) - return -1; - char *buf = tor_malloc(srclen + 4); - memcpy(buf, src, srclen+1); - size_t buflen; - switch (srclen % 4) - { - case 0: - default: - buflen = srclen; - break; - case 1: - tor_free(buf); - return -1; - case 2: - memcpy(buf+srclen, "==", 3); - buflen = srclen + 2; - break; - case 3: - memcpy(buf+srclen, "=", 2); - buflen = srclen + 1; - break; - } - int n = base64_decode((char*)dest, destlen, buf, buflen); - tor_free(buf); - return n; -} - #undef BASE64_OPENSSL_LINELEN /** @{ */ @@ -392,15 +365,9 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) const char *eos = src+srclen; uint32_t n=0; int n_idx=0; - char *dest_orig = dest; + size_t di = 0; - /* Max number of bits == srclen*6. - * Number of bytes required to hold all bits == (srclen*6)/8. - * Yes, we want to round down: anything that hangs over the end of a - * byte is padding. */ - if (destlen < (srclen*3)/4) - return -1; - if (destlen > SIZE_T_CEILING) + if (destlen > INT_MAX) return -1; /* Make sure we leave no uninitialized data in the destination buffer. */ @@ -428,9 +395,11 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) n = (n<<6) | v; if ((++n_idx) == 4) { /* We've accumulated 24 bits in n. Flush them. */ - *dest++ = (n>>16); - *dest++ = (n>>8) & 0xff; - *dest++ = (n) & 0xff; + if (destlen < 3 || di > destlen - 3) + return -1; + dest[di++] = (n>>16); + dest[di++] = (n>>8) & 0xff; + dest[di++] = (n) & 0xff; n_idx = 0; n = 0; } @@ -448,18 +417,21 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) return -1; case 2: /* 12 leftover bits: The last 4 are padding and the first 8 are data. */ - *dest++ = n >> 4; + if (destlen < 1 || di > destlen - 1) + return -1; + dest[di++] = n >> 4; break; case 3: /* 18 leftover bits: The last 2 are padding and the first 16 are data. */ - *dest++ = n >> 10; - *dest++ = n >> 2; + if (destlen < 2 || di > destlen - 2) + return -1; + dest[di++] = n >> 10; + dest[di++] = n >> 2; } - tor_assert((dest-dest_orig) <= (ssize_t)destlen); - tor_assert((dest-dest_orig) <= INT_MAX); + tor_assert(di <= destlen); - return (int)(dest-dest_orig); + return (int)di; } #undef X #undef SP @@ -475,7 +447,8 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen) const char *end; char *cp; - tor_assert(destlen >= srclen*2+1); + tor_assert(srclen < SIZE_T_CEILING / 2 - 1); + tor_assert(destlen >= BASE16_BUFSIZE(srclen)); tor_assert(destlen < SIZE_T_CEILING); /* Make sure we leave no uninitialized data in the destination buffer. */ diff --git a/src/common/util_format.h b/src/common/util_format.h index 20ac711d10..0aefe3a44e 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_UTIL_FORMAT_H @@ -10,6 +10,26 @@ #include "testsupport.h" #include "torint.h" +/** @{ */ +/** These macros don't check for overflow. Use them only for constant inputs + * (like array declarations). The *_LEN macros are the raw encoding lengths + * (without terminating NUL), while the *_BUFSIZE macros count the terminating + * NUL. */ +#define BASE64_LEN(n) (CEIL_DIV((n), 3) * 4) +#define BASE32_LEN(n) (CEIL_DIV((n), 5) * 8) +#define BASE16_LEN(n) ((n) * 2) + +#define BASE64_BUFSIZE(n) (BASE64_LEN(n) + 1) +#define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1) +#define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1) + +#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3)) +#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5)) + +#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1) +#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1) +/** @} */ + #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, @@ -17,8 +37,6 @@ int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); int base64_encode_nopad(char *dest, size_t destlen, const uint8_t *src, size_t srclen); -int base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen); /** Characters that can appear (case-insensitively) in a base32 encoding. */ #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567" @@ -30,5 +48,5 @@ int hex_decode_digit(char c); void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); -#endif +#endif /* !defined(TOR_UTIL_FORMAT_H) */ diff --git a/src/common/util_process.c b/src/common/util_process.c index abda63720c..c2826152e9 100644 --- a/src/common/util_process.c +++ b/src/common/util_process.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -154,5 +154,5 @@ notify_pending_waitpid_callbacks(void) } } -#endif +#endif /* !defined(_WIN32) */ diff --git a/src/common/util_process.h b/src/common/util_process.h index d38301a354..c9aa771b77 100644 --- a/src/common/util_process.h +++ b/src/common/util_process.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,7 +20,7 @@ waitpid_callback_t *set_waitpid_callback(pid_t pid, void (*fn)(int, void *), void *arg); void clear_waitpid_callback(waitpid_callback_t *ent); void notify_pending_waitpid_callbacks(void); -#endif +#endif /* !defined(_WIN32) */ -#endif +#endif /* !defined(TOR_UTIL_PROCESS_H) */ diff --git a/src/common/workqueue.c b/src/common/workqueue.c index e1fb663a2a..42723224d3 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -25,11 +25,19 @@ #include "orconfig.h" #include "compat.h" #include "compat_threads.h" +#include "crypto.h" #include "util.h" #include "workqueue.h" #include "tor_queue.h" #include "torlog.h" +#define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH +#define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW +#define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1) + +TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s); +typedef struct work_tailq_t work_tailq_t; + struct threadpool_s { /** An array of pointers to workerthread_t: one for each running worker * thread. */ @@ -38,8 +46,12 @@ struct threadpool_s { /** Condition variable that we wait on when we have no work, and which * gets signaled when our queue becomes nonempty. */ tor_cond_t condition; - /** Queue of pending work that we have to do. */ - TOR_TAILQ_HEAD(, workqueue_entry_s) work; + /** Queues of pending work that we have to do. The queue with priority + * <b>p</b> is work[p]. */ + work_tailq_t work[WORKQUEUE_N_PRIORITIES]; + + /** Weak RNG, used to decide when to ignore priority. */ + tor_weak_rng_t weak_rng; /** The current 'update generation' of the threadpool. Any thread that is * at an earlier generation needs to run the update function. */ @@ -66,6 +78,11 @@ struct threadpool_s { void *new_thread_state_arg; }; +/** Used to put a workqueue_priority_t value into a bitfield. */ +#define workqueue_priority_bitfield_t ENUM_BF(workqueue_priority_t) +/** Number of bits needed to hold all legal values of workqueue_priority_t */ +#define WORKQUEUE_PRIORITY_BITS 2 + struct workqueue_entry_s { /** The next workqueue_entry_t that's pending on the same thread or * reply queue. */ @@ -76,6 +93,8 @@ struct workqueue_entry_s { struct threadpool_s *on_pool; /** True iff this entry is waiting for a worker to start processing it. */ uint8_t pending; + /** Priority of this entry. */ + workqueue_priority_bitfield_t priority : WORKQUEUE_PRIORITY_BITS; /** Function to run in the worker thread. */ workqueue_reply_t (*fn)(void *state, void *arg); /** Function to run while processing the reply queue. */ @@ -94,9 +113,7 @@ struct replyqueue_s { alert_sockets_t alert; }; -/** A worker thread represents a single thread in a thread pool. To avoid - * contention, each gets its own queue. This breaks the guarantee that that - * queued work will get executed strictly in order. */ +/** A worker thread represents a single thread in a thread pool. */ typedef struct workerthread_s { /** Which thread it this? In range 0..in_pool->n_threads-1 */ int index; @@ -109,6 +126,8 @@ typedef struct workerthread_s { replyqueue_t *reply_queue; /** The current update generation of this thread */ unsigned generation; + /** One over the probability of taking work from a lower-priority queue. */ + int32_t lower_priority_chance; } workerthread_t; static void queue_reply(replyqueue_t *queue, workqueue_entry_t *work); @@ -125,6 +144,7 @@ workqueue_entry_new(workqueue_reply_t (*fn)(void*, void*), ent->fn = fn; ent->reply_fn = reply_fn; ent->arg = arg; + ent->priority = WQ_PRI_HIGH; return ent; } @@ -161,8 +181,9 @@ workqueue_entry_cancel(workqueue_entry_t *ent) int cancelled = 0; void *result = NULL; tor_mutex_acquire(&ent->on_pool->lock); + workqueue_priority_t prio = ent->priority; if (ent->pending) { - TOR_TAILQ_REMOVE(&ent->on_pool->work, ent, next_work); + TOR_TAILQ_REMOVE(&ent->on_pool->work[prio], ent, next_work); cancelled = 1; result = ent->arg; } @@ -180,8 +201,46 @@ workqueue_entry_cancel(workqueue_entry_t *ent) static int worker_thread_has_work(workerthread_t *thread) { - return !TOR_TAILQ_EMPTY(&thread->in_pool->work) || - thread->generation != thread->in_pool->generation; + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + if (!TOR_TAILQ_EMPTY(&thread->in_pool->work[i])) + return 1; + } + return thread->generation != thread->in_pool->generation; +} + +/** Extract the next workqueue_entry_t from the the thread's pool, removing + * it from the relevant queues and marking it as non-pending. + * + * The caller must hold the lock. */ +static workqueue_entry_t * +worker_thread_extract_next_work(workerthread_t *thread) +{ + threadpool_t *pool = thread->in_pool; + work_tailq_t *queue = NULL, *this_queue; + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + this_queue = &pool->work[i]; + if (!TOR_TAILQ_EMPTY(this_queue)) { + queue = this_queue; + if (! tor_weak_random_one_in_n(&pool->weak_rng, + thread->lower_priority_chance)) { + /* Usually we'll just break now, so that we can get out of the loop + * and use the queue where we found work. But with a small + * probability, we'll keep looking for lower priority work, so that + * we don't ignore our low-priority queues entirely. */ + break; + } + } + } + + if (queue == NULL) + return NULL; + + workqueue_entry_t *work = TOR_TAILQ_FIRST(queue); + TOR_TAILQ_REMOVE(queue, work, next_work); + work->pending = 0; + return work; } /** @@ -217,9 +276,9 @@ worker_thread_main(void *thread_) tor_mutex_acquire(&pool->lock); continue; } - work = TOR_TAILQ_FIRST(&pool->work); - TOR_TAILQ_REMOVE(&pool->work, work, next_work); - work->pending = 0; + work = worker_thread_extract_next_work(thread); + if (BUG(work == NULL)) + break; tor_mutex_release(&pool->lock); /* We run the work function without holding the thread lock. This @@ -268,12 +327,14 @@ queue_reply(replyqueue_t *queue, workqueue_entry_t *work) /** Allocate and start a new worker thread to use state object <b>state</b>, * and send responses to <b>replyqueue</b>. */ static workerthread_t * -workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) +workerthread_new(int32_t lower_priority_chance, + void *state, threadpool_t *pool, replyqueue_t *replyqueue) { workerthread_t *thr = tor_malloc_zero(sizeof(workerthread_t)); thr->state = state; thr->reply_queue = replyqueue; thr->in_pool = pool; + thr->lower_priority_chance = lower_priority_chance; if (spawn_func(worker_thread_main, thr) < 0) { //LCOV_EXCL_START @@ -299,24 +360,34 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) * function's responsibility to free the work object. * * On success, return a workqueue_entry_t object that can be passed to - * workqueue_entry_cancel(). On failure, return NULL. + * workqueue_entry_cancel(). On failure, return NULL. (Failure is not + * currently possible, but callers should check anyway.) + * + * Items are executed in a loose priority order -- each thread will usually + * take from the queued work with the highest prioirity, but will occasionally + * visit lower-priority queues to keep them from starving completely. * - * Note that because each thread has its own work queue, work items may not + * Note that because of priorities and thread behavior, work items may not * be executed strictly in order. */ workqueue_entry_t * -threadpool_queue_work(threadpool_t *pool, - workqueue_reply_t (*fn)(void *, void *), - void (*reply_fn)(void *), - void *arg) +threadpool_queue_work_priority(threadpool_t *pool, + workqueue_priority_t prio, + workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg) { + tor_assert(((int)prio) >= WORKQUEUE_PRIORITY_FIRST && + ((int)prio) <= WORKQUEUE_PRIORITY_LAST); + workqueue_entry_t *ent = workqueue_entry_new(fn, reply_fn, arg); ent->on_pool = pool; ent->pending = 1; + ent->priority = prio; tor_mutex_acquire(&pool->lock); - TOR_TAILQ_INSERT_TAIL(&pool->work, ent, next_work); + TOR_TAILQ_INSERT_TAIL(&pool->work[prio], ent, next_work); tor_cond_signal_one(&pool->condition); @@ -325,6 +396,16 @@ threadpool_queue_work(threadpool_t *pool, return ent; } +/** As threadpool_queue_work_priority(), but assumes WQ_PRI_HIGH */ +workqueue_entry_t * +threadpool_queue_work(threadpool_t *pool, + workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg) +{ + return threadpool_queue_work_priority(pool, WQ_PRI_HIGH, fn, reply_fn, arg); +} + /** * Queue a copy of a work item for every thread in a pool. This can be used, * for example, to tell the threads to update some parameter in their states. @@ -388,6 +469,14 @@ threadpool_queue_update(threadpool_t *pool, /** Don't have more than this many threads per pool. */ #define MAX_THREADS 1024 +/** For half of our threads, choose lower priority queues with probability + * 1/N for each of these values. Both are chosen somewhat arbitrarily. If + * CHANCE_PERMISSIVE is too low, then we have a risk of low-priority tasks + * stalling forever. If it's too high, we have a risk of low-priority tasks + * grabbing half of the threads. */ +#define CHANCE_PERMISSIVE 37 +#define CHANCE_STRICT INT32_MAX + /** Launch threads until we have <b>n</b>. */ static int threadpool_start_threads(threadpool_t *pool, int n) @@ -404,8 +493,14 @@ threadpool_start_threads(threadpool_t *pool, int n) sizeof(workerthread_t*), n); while (pool->n_threads < n) { + /* For half of our threads, we'll choose lower priorities permissively; + * for the other half, we'll stick more strictly to higher priorities. + * This keeps slow low-priority tasks from taking over completely. */ + int32_t chance = (pool->n_threads & 1) ? CHANCE_STRICT : CHANCE_PERMISSIVE; + void *state = pool->new_thread_state_fn(pool->new_thread_state_arg); - workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue); + workerthread_t *thr = workerthread_new(chance, + state, pool, pool->reply_queue); if (!thr) { //LCOV_EXCL_START @@ -441,7 +536,15 @@ threadpool_new(int n_threads, pool = tor_malloc_zero(sizeof(threadpool_t)); tor_mutex_init_nonrecursive(&pool->lock); tor_cond_init(&pool->condition); - TOR_TAILQ_INIT(&pool->work); + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + TOR_TAILQ_INIT(&pool->work[i]); + } + { + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(&pool->weak_rng, seed); + } pool->new_thread_state_fn = new_thread_state_fn; pool->new_thread_state_arg = arg; @@ -510,12 +613,13 @@ replyqueue_get_socket(replyqueue_t *rq) void replyqueue_process(replyqueue_t *queue) { - if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { + int r = queue->alert.drain_fn(queue->alert.read_fd); + if (r < 0) { //LCOV_EXCL_START static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, "Failure from drain_fd: %s", - tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); + tor_socket_strerror(-r)); //LCOV_EXCL_STOP } diff --git a/src/common/workqueue.h b/src/common/workqueue.h index 54276767b0..eb885e680d 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_WORKQUEUE_H @@ -16,12 +16,26 @@ typedef struct threadpool_s threadpool_t; typedef struct workqueue_entry_s workqueue_entry_t; /** Possible return value from a work function: */ -typedef enum { +typedef enum workqueue_reply_t { WQ_RPL_REPLY = 0, /** indicates success */ WQ_RPL_ERROR = 1, /** indicates fatal error */ WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */ } workqueue_reply_t; +/** Possible priorities for work. Lower numeric values are more important. */ +typedef enum workqueue_priority_t { + WQ_PRI_HIGH = 0, + WQ_PRI_MED = 1, + WQ_PRI_LOW = 2, +} workqueue_priority_t; + +workqueue_entry_t *threadpool_queue_work_priority(threadpool_t *pool, + workqueue_priority_t prio, + workqueue_reply_t (*fn)(void *, + void *), + void (*reply_fn)(void *), + void *arg); + workqueue_entry_t *threadpool_queue_work(threadpool_t *pool, workqueue_reply_t (*fn)(void *, void *), @@ -45,5 +59,5 @@ replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); tor_socket_t replyqueue_get_socket(replyqueue_t *rq); void replyqueue_process(replyqueue_t *queue); -#endif +#endif /* !defined(TOR_WORKQUEUE_H) */ |