diff options
Diffstat (limited to 'src/common')
79 files changed, 3849 insertions, 1093 deletions
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index b8c5dd4fea..a1c819fffa 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \ - crypto_curve25519.obj curve25519-donna.obj +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \ + tortls.obj crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj diff --git a/src/common/address.c b/src/common/address.c index 2693239146..894e15114c 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -564,8 +564,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name, * and store the result in the <b>outlen</b>-byte buffer at - * <b>out</b>. Return the number of chars written to <b>out</b>, not - * including the trailing \0, on success. Returns -1 on failure. */ + * <b>out</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_addr_to_PTR_name(char *out, size_t outlen, const tor_addr_t *addr) @@ -1198,7 +1198,7 @@ tor_addr_hash(const tor_addr_t *addr) /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; - /* LCOV_EXCL_END */ + /* LCOV_EXCL_STOP */ } } @@ -1781,9 +1781,10 @@ free_interface_address6_list(smartlist_t *addrs) * Returns NULL on failure. * Use free_interface_address6_list to free the returned list. */ -MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, - sa_family_t family, - int include_internal)) +MOCK_IMPL(smartlist_t *, +get_interface_address6_list,(int severity, + sa_family_t family, + int include_internal)) { smartlist_t *addrs; tor_addr_t addr; @@ -2051,7 +2052,8 @@ parse_port_range(const char *port, uint16_t *port_min_out, /** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual), * write it as a string into the <b>buf_len</b>-byte buffer in - * <b>buf</b>. + * <b>buf</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len) diff --git a/src/common/address.h b/src/common/address.h index 0dc6edae37..ce85b3d81d 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/aes.c b/src/common/aes.c index 35c2d1e3a5..73abef143e 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/aes.h b/src/common/aes.h index 1cda53f2fa..6fd9c3ea16 100644 --- a/src/common/aes.h +++ b/src/common/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ diff --git a/src/common/backtrace.c b/src/common/backtrace.c index 61096952d8..0f0fa857bf 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/backtrace.h b/src/common/backtrace.h index b53fd2c668..8cd1dcf06c 100644 --- a/src/common/backtrace.h +++ b/src/common/backtrace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BACKTRACE_H diff --git a/src/common/compat.c b/src/common/compat.c index 4b8b73b4fc..4d110aba35 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1732,19 +1732,24 @@ set_max_file_descriptors(rlim_t limit, int *max_out) if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) { int bad = 1; #ifdef OPEN_MAX - if (errno == EINVAL && OPEN_MAX < rlim.rlim_cur) { + uint64_t try_limit = OPEN_MAX - ULIMIT_BUFFER; + if (errno == EINVAL && try_limit < (uint64_t) rlim.rlim_cur) { /* On some platforms, OPEN_MAX is the real limit, and getrlimit() is * full of nasty lies. I'm looking at you, OSX 10.5.... */ - rlim.rlim_cur = OPEN_MAX; + rlim.rlim_cur = try_limit; if (setrlimit(RLIMIT_NOFILE, &rlim) == 0) { if (rlim.rlim_cur < (rlim_t)limit) { log_warn(LD_CONFIG, "We are limited to %lu file descriptors by " - "OPEN_MAX, and ConnLimit is %lu. Changing ConnLimit; sorry.", - (unsigned long)OPEN_MAX, (unsigned long)limit); + "OPEN_MAX (%lu), and ConnLimit is %lu. Changing " + "ConnLimit; sorry.", + (unsigned long)try_limit, (unsigned long)OPEN_MAX, + (unsigned long)limit); } else { - log_info(LD_CONFIG, "Dropped connection limit to OPEN_MAX (%lu); " - "Apparently, %lu was too high and rlimit lied to us.", - (unsigned long)OPEN_MAX, (unsigned long)rlim.rlim_max); + log_info(LD_CONFIG, "Dropped connection limit to %lu based on " + "OPEN_MAX (%lu); Apparently, %lu was too high and rlimit " + "lied to us.", + (unsigned long)try_limit, (unsigned long)OPEN_MAX, + (unsigned long)rlim.rlim_max); } bad = 0; } @@ -2682,7 +2687,8 @@ static int uname_result_is_set = 0; /** Return a pointer to a description of our platform. */ -MOCK_IMPL(const char *, get_uname, (void)) +MOCK_IMPL(const char *, +get_uname,(void)) { #ifdef HAVE_UNAME struct utsname u; @@ -3255,7 +3261,7 @@ format_win32_error(DWORD err) FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), (LPVOID)&str, 0, NULL); diff --git a/src/common/compat.h b/src/common/compat.h index ee1c9454de..473ad2b957 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_H @@ -414,10 +414,10 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); #endif #ifndef timercmp -/** Replacement for timersub on platforms that do not have it: returns true +/** Replacement for timercmp on platforms that do not have it: returns true * iff the relational operator "op" makes the expression tv1 op tv2 true. * - * Note that while this definition should work for all boolean opeators, some + * Note that while this definition should work for all boolean operators, some * platforms' native timercmp definitions do not support >=, <=, or ==. So * don't use those. */ diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 4a3b1af922..31eb4ac496 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -280,6 +280,15 @@ tor_gettimeofday_cache_set(const struct timeval *tv) tor_assert(tv); memcpy(&cached_time_hires, tv, sizeof(*tv)); } + +/** For testing: called post-fork to make libevent reinitialize + * kernel structures. */ +void +tor_libevent_postfork(void) +{ + int r = event_reinit(tor_libevent_get_base()); + tor_assert(r == 0); +} #endif #endif diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index c2e34764e4..904938415c 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_LIBEVENT_H @@ -54,6 +54,7 @@ void tor_gettimeofday_cached(struct timeval *tv); void tor_gettimeofday_cache_clear(void); #ifdef TOR_UNIT_TESTS void tor_gettimeofday_cache_set(const struct timeval *tv); +void tor_libevent_postfork(void); #endif #ifdef COMPAT_LIBEVENT_PRIVATE diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h index 1bfe188075..2b94fe5b4e 100644 --- a/src/common/compat_openssl.h +++ b/src/common/compat_openssl.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_OPENSSL_H diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c index c1ae66c1d2..8e4b833573 100644 --- a/src/common/compat_pthreads.c +++ b/src/common/compat_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_rust.c b/src/common/compat_rust.c new file mode 100644 index 0000000000..366fd4037b --- /dev/null +++ b/src/common/compat_rust.c @@ -0,0 +1,39 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rust_compat.c + * \brief Rust FFI compatibility functions and helpers. This file is only built + * if Rust is not used. + **/ + +#include "compat_rust.h" +#include "util.h" + +/** + * Free storage pointed to by <b>str</b>, and itself. + */ +void +rust_str_free(rust_str_t str) +{ + char *s = (char *)str; + tor_free(s); +} + +/** + * Return zero-terminated contained string. + */ +const char * +rust_str_get(const rust_str_t str) +{ + return (const char *)str; +} + +/* If we were using Rust, we'd say so on startup. */ +rust_str_t +rust_welcome_string(void) +{ + char *s = tor_malloc_zero(1); + return (rust_str_t)s; +} + diff --git a/src/common/compat_rust.h b/src/common/compat_rust.h new file mode 100644 index 0000000000..752a29b56c --- /dev/null +++ b/src/common/compat_rust.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rust_compat.h + * \brief Headers for rust_compat.c + **/ + +#ifndef TOR_RUST_COMPAT_H +#define TOR_RUST_COMPAT_H + +#include "torint.h" + +/** + * Strings allocated in Rust must be freed from Rust code again. Let's make + * it less likely to accidentally mess up and call tor_free() on it, because + * currently it'll just work but might break at any time. + */ +typedef uintptr_t rust_str_t; + +void rust_str_free(rust_str_t); + +const char *rust_str_get(const rust_str_t); + +rust_str_t rust_welcome_string(void); + +#endif + diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index f4809060d6..c593e9af8d 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -94,51 +94,73 @@ in_main_thread(void) } #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE) -/* As write(), but retry on EINTR */ +/* As write(), but retry on EINTR, and return the negative error code on + * error. */ static int write_ni(int fd, const void *buf, size_t n) { int r; again: r = (int) write(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } -/* As read(), but retry on EINTR */ +/* As read(), but retry on EINTR, and return the negative error code on error. + */ static int read_ni(int fd, void *buf, size_t n) { int r; again: r = (int) read(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } #endif -/** As send(), but retry on EINTR. */ +/** As send(), but retry on EINTR, and return the negative error code on + * error. */ static int send_ni(int fd, const void *buf, size_t n, int flags) { int r; again: r = (int) send(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } -/** As recv(), but retry on EINTR. */ +/** As recv(), but retry on EINTR, and return the negative error code on + * error. */ static int recv_ni(int fd, void *buf, size_t n, int flags) { int r; again: r = (int) recv(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } @@ -149,7 +171,7 @@ eventfd_alert(int fd) { uint64_t u = 1; int r = write_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) + if (r < 0 && -r != EAGAIN) return -1; return 0; } @@ -160,8 +182,8 @@ eventfd_drain(int fd) { uint64_t u = 0; int r = read_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return r; return 0; } #endif @@ -172,8 +194,8 @@ static int pipe_alert(int fd) { ssize_t r = write_ni(fd, "x", 1); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return (int)r; return 0; } @@ -188,7 +210,7 @@ pipe_drain(int fd) r = read_ni(fd, buf, sizeof(buf)); } while (r > 0); if (r < 0 && errno != EAGAIN) - return -1; + return -errno; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } @@ -200,13 +222,13 @@ static int sock_alert(tor_socket_t fd) { ssize_t r = send_ni(fd, "x", 1, 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; return 0; } /** Drain all the input from a socket <b>fd</b>, and ignore it. Return 0 on - * success, -1 on error. */ + * success, -errno on error. */ static int sock_drain(tor_socket_t fd) { @@ -215,8 +237,8 @@ sock_drain(tor_socket_t fd) do { r = recv_ni(fd, buf, sizeof(buf), 0); } while (r > 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } @@ -330,3 +352,49 @@ alert_sockets_close(alert_sockets_t *socks) socks->read_fd = socks->write_fd = -1; } +/* + * XXXX We might be smart to move to compiler intrinsics or real atomic + * XXXX operations at some point. But not yet. + * + */ + +/** Initialize a new atomic counter with the value 0 */ +void +atomic_counter_init(atomic_counter_t *counter) +{ + memset(counter, 0, sizeof(*counter)); + tor_mutex_init_nonrecursive(&counter->mutex); +} +/** Clean up all resources held by an atomic counter. */ +void +atomic_counter_destroy(atomic_counter_t *counter) +{ + tor_mutex_uninit(&counter->mutex); + memset(counter, 0, sizeof(*counter)); +} +/** Add a value to an atomic counter. */ +void +atomic_counter_add(atomic_counter_t *counter, size_t add) +{ + tor_mutex_acquire(&counter->mutex); + counter->val += add; + tor_mutex_release(&counter->mutex); +} +/** Subtract a value from an atomic counter. */ +void +atomic_counter_sub(atomic_counter_t *counter, size_t sub) +{ + // this relies on unsigned overflow, but that's fine. + atomic_counter_add(counter, -sub); +} +/** Return the current value of an atomic counter */ +size_t +atomic_counter_get(atomic_counter_t *counter) +{ + size_t val; + tor_mutex_acquire(&counter->mutex); + val = counter->val; + tor_mutex_release(&counter->mutex); + return val; +} + diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h index 171a9f93ff..9fa3d0d0b7 100644 --- a/src/common/compat_threads.h +++ b/src/common/compat_threads.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_THREADS_H @@ -147,5 +147,19 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal); */ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value); +/** + * Atomic counter type; holds a size_t value. + */ +typedef struct atomic_counter_t { + tor_mutex_t mutex; + size_t val; +} atomic_counter_t; + +void atomic_counter_init(atomic_counter_t *counter); +void atomic_counter_destroy(atomic_counter_t *counter); +void atomic_counter_add(atomic_counter_t *counter, size_t add); +void atomic_counter_sub(atomic_counter_t *counter, size_t sub); +size_t atomic_counter_get(atomic_counter_t *counter); + #endif diff --git a/src/common/compat_time.c b/src/common/compat_time.c index d044bbe1d7..2ccaa36e49 100644 --- a/src/common/compat_time.c +++ b/src/common/compat_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_time.h b/src/common/compat_time.h index 2262446e57..90194c5ebc 100644 --- a/src/common/compat_time.h +++ b/src/common/compat_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c index 735be4ad17..915368b94f 100644 --- a/src/common/compat_winthreads.c +++ b/src/common/compat_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compress.c b/src/common/compress.c new file mode 100644 index 0000000000..7926faaa60 --- /dev/null +++ b/src/common/compress.c @@ -0,0 +1,651 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.c + * \brief Common compression API. + **/ + +#include "orconfig.h" + +#include <stdlib.h> +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include "torint.h" + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" +#include "compress_none.h" +#include "compress_zlib.h" +#include "compress_zstd.h" + +/** Total number of bytes allocated for compression state overhead. */ +static atomic_counter_t total_compress_allocation; + +/** @{ */ +/* These macros define the maximum allowable compression factor. Anything of + * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to + * have an uncompression factor (uncompressed size:compressed size ratio) of + * any greater than MAX_UNCOMPRESSION_FACTOR. + * + * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to + * be small to limit the attack multiplier, but we also want it to be large + * enough so that no legitimate document --even ones we might invent in the + * future -- ever compresses by a factor of greater than + * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably + * large range of possible values. IMO, anything over 8 is probably safe; IMO + * anything under 50 is probably sufficient. + */ +#define MAX_UNCOMPRESSION_FACTOR 25 +#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ + +/** Return true if uncompressing an input of size <b>in_size</b> to an input of + * size at least <b>size_out</b> looks like a compression bomb. */ +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out) +{ + if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) + return 0; + + return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); +} + +/** Guess the size that <b>in_len</b> will be after compression or + * decompression. */ +static size_t +guess_compress_size(int compress, compress_method_t method, + compression_level_t compression_level, + size_t in_len) +{ + // ignore these for now. + (void)compression_level; + if (method == NO_METHOD) { + /* Guess that we'll need an extra byte, to avoid a needless realloc + * for nul-termination */ + return (in_len < SIZE_MAX) ? in_len + 1 : in_len; + } + + /* Always guess a factor of 2. */ + if (compress) { + in_len /= 2; + } else { + if (in_len < SIZE_T_CEILING/2) + in_len *= 2; + } + return MAX(in_len, 1024); +} + +/** Internal function to implement tor_compress/tor_uncompress, depending on + * whether <b>compress</b> is set. All arguments are as for tor_compress or + * tor_uncompress. */ +static int +tor_compress_impl(int compress, + char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + compression_level_t compression_level, + int complete_only, + int protocol_warn_level) +{ + tor_compress_state_t *stream; + int rv; + + stream = tor_compress_new(compress, method, compression_level); + + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + return -1; + } + + size_t in_len_orig = in_len; + size_t out_remaining, out_alloc; + char *outptr; + + out_remaining = out_alloc = + guess_compress_size(compress, method, compression_level, in_len); + *out = outptr = tor_malloc(out_remaining); + + const int finish = complete_only || compress; + + while (1) { + switch (tor_compress_process(stream, + &outptr, &out_remaining, + &in, &in_len, finish)) { + case TOR_COMPRESS_DONE: + if (in_len == 0 || compress) { + goto done; + } else { + // More data is present, and we're decompressing. So we may need to + // reinitialize the stream if we are handling multiple concatenated + // inputs. + tor_compress_free(stream); + stream = tor_compress_new(compress, method, compression_level); + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + goto err; + } + } + break; + case TOR_COMPRESS_OK: + if (compress || complete_only) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "Unexpected %s while %scompressing", + complete_only?"end of input":"result", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + goto err; + } else { + if (in_len == 0) { + goto done; + } + } + break; + case TOR_COMPRESS_BUFFER_FULL: { + if (!compress && outptr < *out+out_alloc) { + // A buffer error in this case means that we have a problem + // with our input. + log_fn(protocol_warn_level, LD_PROTOCOL, + "Possible truncated or corrupt compressed data"); + goto err; + } + if (out_alloc >= SIZE_T_CEILING / 2) { + log_warn(LD_GENERAL, "While %scompresing data: ran out of space.", + compress?"":"un"); + goto err; + } + if (!compress && + tor_compress_is_compression_bomb(in_len_orig, out_alloc)) { + // This should already have been caught down in the backend logic. + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + const size_t offset = outptr - *out; + out_alloc *= 2; + *out = tor_realloc(*out, out_alloc); + outptr = *out + offset; + out_remaining = out_alloc - offset; + break; + } + case TOR_COMPRESS_ERROR: + log_fn(protocol_warn_level, LD_GENERAL, + "Error while %scompresing data: bad input?", + compress?"":"un"); + goto err; // bad data. + default: + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + } + done: + *out_len = outptr - *out; + if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a " + "compression bomb."); + goto err; + } + if (!compress) { + // NUL-terminate our output. + if (out_alloc == *out_len) + *out = tor_realloc(*out, out_alloc + 1); + (*out)[*out_len] = '\0'; + } + rv = 0; + goto out; + + err: + tor_free(*out); + *out_len = 0; + rv = -1; + goto out; + + out: + tor_compress_free(stream); + return rv; +} + +/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly + * allocated buffer, using the method described in <b>method</b>. Store the + * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. + * Return 0 on success, -1 on failure. + */ +int +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + return tor_compress_impl(1, out, out_len, in, in_len, method, + BEST_COMPRESSION, + 1, LOG_WARN); +} + +/** Given zero or more compressed strings of total length <b>in_len</b> bytes + * at <b>in</b>, uncompress them into a newly allocated buffer, using the + * method described in <b>method</b>. Store the uncompressed string in + * *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 on + * failure. + * + * If any bytes are written to <b>out</b>, an extra byte NUL is always + * written at the end, but not counted in <b>out_len</b>. This is a + * safety feature to ensure that the output can be treated as a + * NUL-terminated string -- though of course, callers should check + * out_len anyway. + * + * If <b>complete_only</b> is true, we consider a truncated input as a + * failure; otherwise we decompress as much as we can. Warn about truncated + * or corrupt inputs at <b>protocol_warn_level</b>. + */ +int +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + return tor_compress_impl(0, out, out_len, in, in_len, method, + BEST_COMPRESSION, + complete_only, protocol_warn_level); +} + +/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely + * to be compressed or not. If it is, return the likeliest compression method. + * Otherwise, return UNKNOWN_METHOD. + */ +compress_method_t +detect_compression_method(const char *in, size_t in_len) +{ + if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { + return GZIP_METHOD; + } else if (in_len > 2 && (in[0] & 0x0f) == 8 && + (ntohs(get_uint16(in)) % 31) == 0) { + return ZLIB_METHOD; + } else if (in_len > 2 && + fast_memeq(in, "\x5d\x00\x00", 3)) { + return LZMA_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) { + return ZSTD_METHOD; + } else { + return UNKNOWN_METHOD; + } +} + +/** Return 1 if a given <b>method</b> is supported; otherwise 0. */ +int +tor_compress_supports_method(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_method_supported(); + case LZMA_METHOD: + return tor_lzma_method_supported(); + case ZSTD_METHOD: + return tor_zstd_method_supported(); + case NO_METHOD: + return 1; + case UNKNOWN_METHOD: + default: + return 0; + } +} + +/** + * Return a bitmask of the supported compression types, where 1<<m is + * set in the bitmask if and only if compression with method <b>m</b> is + * supported. + */ +unsigned +tor_compress_get_supported_method_bitmask(void) +{ + static unsigned supported = 0; + if (supported == 0) { + compress_method_t m; + for (m = NO_METHOD; m <= UNKNOWN_METHOD; ++m) { + if (tor_compress_supports_method(m)) { + supported |= (1u << m); + } + } + } + return supported; +} + +/** Table of compression method names. These should have an "x-" prefix, + * if they are not listed in the IANA content coding registry. */ +static const struct { + const char *name; + compress_method_t method; +} compression_method_names[] = { + { "gzip", GZIP_METHOD }, + { "deflate", ZLIB_METHOD }, + // We call this "x-tor-lzma" rather than "x-lzma", because we impose a + // lower maximum memory usage on the decoding side. + { "x-tor-lzma", LZMA_METHOD }, + { "x-zstd" , ZSTD_METHOD }, + { "identity", NO_METHOD }, + + /* Later entries in this table are not canonical; these are recognized but + * not emitted. */ + { "x-gzip", GZIP_METHOD }, +}; + +/** Return the canonical string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (method == compression_method_names[i].method) + return compression_method_names[i].name; + } + return NULL; +} + +/** Table of compression human readable method names. */ +static const struct { + compress_method_t method; + const char *name; +} compression_method_human_names[] = { + { NO_METHOD, "uncompressed" }, + { GZIP_METHOD, "gzipped" }, + { ZLIB_METHOD, "deflated" }, + { LZMA_METHOD, "LZMA compressed" }, + { ZSTD_METHOD, "Zstandard compressed" }, + { UNKNOWN_METHOD, "unknown encoding" }, +}; + +/** Return a human readable string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_human_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) { + if (method == compression_method_human_names[i].method) + return compression_method_human_names[i].name; + } + return NULL; +} + +/** Return the compression method represented by the string <b>name</b>, or + * UNKNOWN_METHOD if the string isn't recognized. */ +compress_method_t +compression_method_get_by_name(const char *name) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (!strcmp(compression_method_names[i].name, name)) + return compression_method_names[i].method; + } + return UNKNOWN_METHOD; +} + +/** Return a string representation of the version of the library providing the + * compression method given in <b>method</b>. Returns NULL if <b>method</b> is + * unknown or unsupported. */ +const char * +tor_compress_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_version_str(); + case LZMA_METHOD: + return tor_lzma_get_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return a string representation of the version of the library, found at + * compile time, providing the compression method given in <b>method</b>. + * Returns NULL if <b>method</b> is unknown or unsupported. */ +const char * +tor_compress_header_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_header_version_str(); + case LZMA_METHOD: + return tor_lzma_get_header_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_header_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return the approximate number of bytes allocated for all + * supported compression schemas. */ +size_t +tor_compress_get_total_allocation(void) +{ + return atomic_counter_get(&total_compress_allocation) + + tor_zlib_get_total_allocation() + + tor_lzma_get_total_allocation() + + tor_zstd_get_total_allocation(); +} + +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { + compress_method_t method; /**< The compression method. */ + + union { + tor_zlib_compress_state_t *zlib_state; + tor_lzma_compress_state_t *lzma_state; + tor_zstd_compress_state_t *zstd_state; + } u; /**< Compression backend state. */ +}; + +/** Construct and return a tor_compress_state_t object using <b>method</b>. If + * <b>compress</b>, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress, compress_method_t method, + compression_level_t compression_level) +{ + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + break; + } + case LZMA_METHOD: { + tor_lzma_compress_state_t *lzma_state = + tor_lzma_compress_new(compress, method, compression_level); + + if (lzma_state == NULL) + goto err; + + state->u.lzma_state = lzma_state; + break; + } + case ZSTD_METHOD: { + tor_zstd_compress_state_t *zstd_state = + tor_zstd_compress_new(compress, method, compression_level); + + if (zstd_state == NULL) + goto err; + + state->u.zstd_state = zstd_state; + break; + } + case NO_METHOD: { + break; + } + case UNKNOWN_METHOD: + goto err; + } + + atomic_counter_add(&total_compress_allocation, + sizeof(tor_compress_state_t)); + return state; + + err: + tor_free(state); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + tor_assert(state != NULL); + const size_t in_len_orig = *in_len; + const size_t out_len_orig = *out_len; + tor_compress_output_t rv; + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + rv = tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + break; + case LZMA_METHOD: + rv =tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); + break; + case ZSTD_METHOD: + rv = tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); + break; + case NO_METHOD: + rv = tor_cnone_compress_process(out, out_len, in, in_len, + finish); + break; + default: + case UNKNOWN_METHOD: + goto err; + } + if (BUG((rv == TOR_COMPRESS_OK) && + *in_len == in_len_orig && + *out_len == out_len_orig)) { + return TOR_COMPRESS_ERROR; + } + + return rv; + err: + return TOR_COMPRESS_ERROR; +} + +/** Deallocate <b>state</b>. */ +void +tor_compress_free(tor_compress_state_t *state) +{ + if (state == NULL) + return; + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + tor_zlib_compress_free(state->u.zlib_state); + break; + case LZMA_METHOD: + tor_lzma_compress_free(state->u.lzma_state); + break; + case ZSTD_METHOD: + tor_zstd_compress_free(state->u.zstd_state); + break; + case NO_METHOD: + break; + case UNKNOWN_METHOD: + break; + } + + atomic_counter_sub(&total_compress_allocation, + sizeof(tor_compress_state_t)); + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_compress_state_size(const tor_compress_state_t *state) +{ + tor_assert(state != NULL); + + size_t size = sizeof(tor_compress_state_t); + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + size += tor_zlib_compress_state_size(state->u.zlib_state); + break; + case LZMA_METHOD: + size += tor_lzma_compress_state_size(state->u.lzma_state); + break; + case ZSTD_METHOD: + size += tor_zstd_compress_state_size(state->u.zstd_state); + break; + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } + + return size; +} + +/** Initialize all compression modules. */ +void +tor_compress_init(void) +{ + atomic_counter_init(&total_compress_allocation); + + tor_zlib_init(); + tor_lzma_init(); + tor_zstd_init(); +} + diff --git a/src/common/compress.h b/src/common/compress.h new file mode 100644 index 0000000000..7c0dc14061 --- /dev/null +++ b/src/common/compress.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.h + * \brief Headers for compress.c + **/ + +#ifndef TOR_COMPRESS_H +#define TOR_COMPRESS_H + +/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and + * GZIP_METHOD is guaranteed to be supported by the compress/uncompress + * functions here. Call tor_compress_supports_method() to check if a given + * compression schema is supported by Tor. */ +typedef enum { + NO_METHOD=0, // This method must be first. + GZIP_METHOD=1, + ZLIB_METHOD=2, + LZMA_METHOD=3, + ZSTD_METHOD=4, + UNKNOWN_METHOD=5, // This method must be last. Add new ones in the middle. +} compress_method_t; + +/** + * Enumeration to define tradeoffs between memory usage and compression level. + * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most + * memory. + **/ +typedef enum { + BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION +} compression_level_t; + +int tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +compress_method_t detect_compression_method(const char *in, size_t in_len); + +int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); + +int tor_compress_supports_method(compress_method_t method); +unsigned tor_compress_get_supported_method_bitmask(void); +const char * compression_method_get_name(compress_method_t method); +const char *compression_method_get_human_name(compress_method_t method); +compress_method_t compression_method_get_by_name(const char *name); + +const char *tor_compress_version_str(compress_method_t method); + +const char *tor_compress_header_version_str(compress_method_t method); + +size_t tor_compress_get_total_allocation(void); + +/** Return values from tor_compress_process; see that function's documentation + * for details. */ +typedef enum { + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; + +/** Internal state for an incremental compression/decompression. */ +typedef struct tor_compress_state_t tor_compress_state_t; + +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); + +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free(tor_compress_state_t *state); + +size_t tor_compress_state_size(const tor_compress_state_t *state); + +void tor_compress_init(void); + +#endif // TOR_COMPRESS_H. + diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c new file mode 100644 index 0000000000..b5393a6ba6 --- /dev/null +++ b/src/common/compress_lzma.c @@ -0,0 +1,351 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.c + * \brief Compression backend for LZMA. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" + +#ifdef HAVE_LZMA +#include <lzma.h> +#endif + +/** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */ +#define MEMORY_LIMIT (16 * 1024 * 1024) + +/** Total number of bytes allocated for LZMA state. */ +static atomic_counter_t total_lzma_allocation; + +#ifdef HAVE_LZMA +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 6; + case MEDIUM_COMPRESSION: return 4; + case LOW_COMPRESSION: return 2; + } +} + +/** Convert a given <b>error</b> to a human readable error string. */ +static const char * +lzma_error_str(lzma_ret error) +{ + switch (error) { + case LZMA_OK: + return "Operation completed successfully"; + case LZMA_STREAM_END: + return "End of stream"; + case LZMA_NO_CHECK: + return "Input stream lacks integrity check"; + case LZMA_UNSUPPORTED_CHECK: + return "Unable to calculate integrity check"; + case LZMA_GET_CHECK: + return "Integrity check available"; + case LZMA_MEM_ERROR: + return "Unable to allocate memory"; + case LZMA_MEMLIMIT_ERROR: + return "Memory limit reached"; + case LZMA_FORMAT_ERROR: + return "Unknown file format"; + case LZMA_OPTIONS_ERROR: + return "Unsupported options"; + case LZMA_DATA_ERROR: + return "Corrupt input data"; + case LZMA_BUF_ERROR: + return "Unable to progress"; + case LZMA_PROG_ERROR: + return "Programming error"; + default: + return "Unknown LZMA error"; + } +} +#endif // HAVE_LZMA. + +/** Return 1 if LZMA compression is supported; otherwise 0. */ +int +tor_lzma_method_supported(void) +{ +#ifdef HAVE_LZMA + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of liblzma. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_version_str(void) +{ +#ifdef HAVE_LZMA + return lzma_version_string(); +#else + return NULL; +#endif +} + +/** Return a string representation of the version of liblzma used at + * compilation time. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_header_version_str(void) +{ +#ifdef HAVE_LZMA + return LZMA_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal LZMA state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_lzma_compress_state_t { +#ifdef HAVE_LZMA + lzma_stream stream; /**< The LZMA stream. */ +#endif + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_LZMA +/** Return an approximate number of bytes stored in memory to hold the LZMA + * encoder/decoder state. */ +static size_t +tor_lzma_state_size_precalc(int compress, compression_level_t level) +{ + uint64_t memory_usage; + + if (compress) + memory_usage = lzma_easy_encoder_memusage(memory_level(level)); + else + memory_usage = lzma_easy_decoder_memusage(memory_level(level)); + + if (memory_usage == UINT64_MAX) { + log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s", + compress ? "encoder" : "decoder"); + goto err; + } + + if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX) + memory_usage = SIZE_MAX; + else + memory_usage += sizeof(tor_lzma_compress_state_t); + + return (size_t)memory_usage; + + err: + return 0; +} +#endif // HAVE_LZMA. + +/** Construct and return a tor_lzma_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == LZMA_METHOD); + +#ifdef HAVE_LZMA + tor_lzma_compress_state_t *result; + lzma_ret retval; + lzma_options_lzma stream_options; + + // Note that we do not explicitly initialize the lzma_stream object here, + // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is + // also what `tor_malloc_zero()` does. + result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); + result->compress = compress; + result->allocation = tor_lzma_state_size_precalc(compress, level); + + if (compress) { + lzma_lzma_preset(&stream_options, memory_level(level)); + + retval = lzma_alone_encoder(&result->stream, &stream_options); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } else { + retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } + + atomic_counter_add(&total_lzma_allocation, result->allocation); + return result; + + err: + tor_free(result); + return NULL; +#else // HAVE_LZMA. + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif // HAVE_LZMA. +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_LZMA + lzma_ret retval; + lzma_action action; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + state->stream.next_in = (unsigned char *)*in; + state->stream.avail_in = *in_len; + state->stream.next_out = (unsigned char *)*out; + state->stream.avail_out = *out_len; + + action = finish ? LZMA_FINISH : LZMA_RUN; + + retval = lzma_code(&state->stream, action); + + state->input_so_far += state->stream.next_in - ((unsigned char *)*in); + state->output_so_far += state->stream.next_out - ((unsigned char *)*out); + + *out = (char *)state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *)state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (retval) { + case LZMA_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_OK; + + case LZMA_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + + return TOR_COMPRESS_BUFFER_FULL; + + case LZMA_STREAM_END: + return TOR_COMPRESS_DONE; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + lzma_error_str(retval)); + return TOR_COMPRESS_ERROR; + } +#else // HAVE_LZMA. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + return TOR_COMPRESS_ERROR; +#endif // HAVE_LZMA. +} + +/** Deallocate <b>state</b>. */ +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_lzma_allocation, state->allocation); + +#ifdef HAVE_LZMA + lzma_end(&state->stream); +#endif + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all LZMA states. */ +size_t +tor_lzma_get_total_allocation(void) +{ + return atomic_counter_get(&total_lzma_allocation); +} + +/** Initialize the lzma module */ +void +tor_lzma_init(void) +{ + atomic_counter_init(&total_lzma_allocation); +} + diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h new file mode 100644 index 0000000000..1433c89f88 --- /dev/null +++ b/src/common/compress_lzma.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.h + * \brief Header for compress_lzma.c + **/ + +#ifndef TOR_COMPRESS_LZMA_H +#define TOR_COMPRESS_LZMA_H + +int tor_lzma_method_supported(void); + +const char *tor_lzma_get_version_str(void); + +const char *tor_lzma_get_header_version_str(void); + +/** Internal state for an incremental LZMA compression/decompression. */ +typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; + +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_lzma_compress_free(tor_lzma_compress_state_t *state); + +size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); + +size_t tor_lzma_get_total_allocation(void); + +void tor_lzma_init(void); + +#endif // TOR_COMPRESS_LZMA_H. + diff --git a/src/common/compress_none.c b/src/common/compress_none.c new file mode 100644 index 0000000000..34314e4af7 --- /dev/null +++ b/src/common/compress_none.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.c + * \brief Compression backend for identity compression. + * + * We actually define this backend so that we can treat the identity transform + * as another case of compression. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_none.h" + +/** Transfer some bytes using the identity transformation. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + size_t n_to_copy = MIN(*in_len, *out_len); + + memcpy(*out, *in, n_to_copy); + *out += n_to_copy; + *in += n_to_copy; + *out_len -= n_to_copy; + *in_len -= n_to_copy; + if (*in_len == 0) { + return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK; + } else { + return TOR_COMPRESS_BUFFER_FULL; + } +} + diff --git a/src/common/compress_none.h b/src/common/compress_none.h new file mode 100644 index 0000000000..d1ebb4b625 --- /dev/null +++ b/src/common/compress_none.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.h + * \brief Header for compress_none.c + **/ + +#ifndef TOR_COMPRESS_NONE_H +#define TOR_COMPRESS_NONE_H + +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +#endif // TOR_COMPRESS_NONE_H. + diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000000..284542e885 --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,304 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include <zlib.h> + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static atomic_counter_t total_zlib_allocation; + +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: return 9; + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + +/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return 1 if zlib/gzip compression is supported; otherwise 0. */ +int +tor_zlib_method_supported(void) +{ + /* We currently always support zlib/gzip, but we keep this function around in + * case we some day decide to deprecate zlib/gzip support. + */ + return 1; +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits <b>windowBits</b> and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress_, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress_) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = BEST_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress_; + bits = method_bits(method, compression_level); + memlevel = memory_level(compression_level); + if (compress_) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); + + atomic_counter_add(&total_zlib_allocation, out->allocation); + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + if (*in_len > UINT_MAX || + *out_len > UINT_MAX) { + return TOR_COMPRESS_ERROR; + } + + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : "<no message>"); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate <b>state</b>. */ +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zlib_allocation, state->allocation); + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return atomic_counter_get(&total_zlib_allocation); +} + +/** Set up global state for the zlib module */ +void +tor_zlib_init(void) +{ + atomic_counter_init(&total_zlib_allocation); +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000000..df5c196ac7 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +int tor_zlib_method_supported(void); + +const char *tor_zlib_get_version_str(void); + +const char *tor_zlib_get_header_version_str(void); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zlib_compress_free(tor_zlib_compress_state_t *state); + +size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t tor_zlib_get_total_allocation(void); + +void tor_zlib_init(void); + +#endif // TOR_COMPRESS_ZLIB_H. + diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c new file mode 100644 index 0000000000..94974dec06 --- /dev/null +++ b/src/common/compress_zstd.c @@ -0,0 +1,430 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.c + * \brief Compression backend for Zstandard. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zstd.h" + +#ifdef HAVE_ZSTD +#include <zstd.h> +#endif + +/** Total number of bytes allocated for Zstandard state. */ +static atomic_counter_t total_zstd_allocation; + +#ifdef HAVE_ZSTD +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 8; + case LOW_COMPRESSION: return 7; + } +} +#endif // HAVE_ZSTD. + +/** Return 1 if Zstandard compression is supported; otherwise 0. */ +int +tor_zstd_method_supported(void) +{ +#ifdef HAVE_ZSTD + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of libzstd. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_version_str(void) +{ +#ifdef HAVE_ZSTD + static char version_str[16]; + size_t version_number; + + version_number = ZSTD_versionNumber(); + tor_snprintf(version_str, sizeof(version_str), + "%lu.%lu.%lu", + version_number / 10000 % 100, + version_number / 100 % 100, + version_number % 100); + + return version_str; +#else + return NULL; +#endif +} + +/** Return a string representation of the version of the version of libzstd + * used at compilation time. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_header_version_str(void) +{ +#ifdef HAVE_ZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal Zstandard state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zstd_compress_state_t { +#ifdef HAVE_ZSTD + union { + /** Compression stream. Used when <b>compress</b> is true. */ + ZSTD_CStream *compress_stream; + /** Decompression stream. Used when <b>compress</b> is false. */ + ZSTD_DStream *decompress_stream; + } u; /**< Zstandard stream objects. */ +#endif // HAVE_ZSTD. + + int compress; /**< True if we are compressing; false if we are inflating */ + int have_called_end; /**< True if we are compressing and we've called + * ZSTD_endStream */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_ZSTD +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ + tor_assert(preset > 0); + + size_t memory_usage = sizeof(tor_zstd_compress_state_t); + + // The Zstandard library provides a number of functions that would be useful + // here, but they are, unfortunately, still considered experimental and are + // thus only available in libzstd if we link against the library statically. + // + // The code in this function tries to approximate the calculations without + // being able to use the following: + // + // - We do not have access to neither the internal members of ZSTD_CStream + // and ZSTD_DStream and their internal context objects. + // + // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they + // are unexposed. + // + // In the future it might be useful to check if libzstd have started + // providing these functions in a stable manner and simplify this function. + if (compress) { + // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine: + memory_usage += 192; + + // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to + // variables that are not exposed via the public API. We use a _very_ + // simplified function to calculate the estimated amount of bytes used in + // this struct. + // memory_usage += (preset - 0.5) * 1024 * 1024; + memory_usage += (preset * 1024 * 1024) - (512 * 1024); + // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes. + // - stream->outBuffSize: 128 KB: + memory_usage += 128 * 1024; + // - stream->inBuffSize: 2048 KB: + memory_usage += 2048 * 1024; + } else { + // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine: + memory_usage += 208; + // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB. + memory_usage += 150 * 1024; + + // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes. + // - stream->inBuffSize: 0 KB. + // - stream->outBuffSize: 0 KB. + } + + return memory_usage; +} +#endif // HAVE_ZSTD. + +/** Construct and return a tor_zstd_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == ZSTD_METHOD); + +#ifdef HAVE_ZSTD + const int preset = memory_level(level); + tor_zstd_compress_state_t *result; + size_t retval; + + result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); + result->compress = compress; + result->allocation = tor_zstd_state_size_precalc(compress, preset); + + if (compress) { + result->u.compress_stream = ZSTD_createCStream(); + + if (result->u.compress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initCStream(result->u.compress_stream, preset); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } else { + result->u.decompress_stream = ZSTD_createDStream(); + + if (result->u.decompress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initDStream(result->u.decompress_stream); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } + + atomic_counter_add(&total_zstd_allocation, result->allocation); + return result; + + err: + if (compress) { + ZSTD_freeCStream(result->u.compress_stream); + } else { + ZSTD_freeDStream(result->u.decompress_stream); + } + + tor_free(result); + return NULL; +#else // HAVE_ZSTD. + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif // HAVE_ZSTD. +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_ZSTD + size_t retval; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + ZSTD_inBuffer input = { *in, *in_len, 0 }; + ZSTD_outBuffer output = { *out, *out_len, 0 }; + + if (BUG(finish == 0 && state->have_called_end)) { + finish = 1; + } + + if (state->compress) { + if (! state->have_called_end) + retval = ZSTD_compressStream(state->u.compress_stream, + &output, &input); + else + retval = 0; + } else { + retval = ZSTD_decompressStream(state->u.decompress_stream, + &output, &input); + } + + state->input_so_far += input.pos; + state->output_so_far += output.pos; + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + *in = (char *)input.src + input.pos; + *in_len = input.size - input.pos; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (state->compress && !state->have_called_end) { + retval = ZSTD_flushStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // ZSTD_flushStream returns 0 if the frame is done, or >0 if it + // is incomplete. + if (retval > 0) { + return TOR_COMPRESS_BUFFER_FULL; + } + } + + if (!finish) { + // The caller says we're not done with the input, so no need to write an + // epilogue. + return TOR_COMPRESS_OK; + } else if (state->compress) { + if (*in_len) { + // We say that we're not done with the input, so we can't write an + // epilogue. + return TOR_COMPRESS_OK; + } + + retval = ZSTD_endStream(state->u.compress_stream, &output); + state->have_called_end = 1; + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to write " + "epilogue: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // endStream returns the number of bytes that is needed to write the + // epilogue. + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_DONE; + } else /* if (!state->compress) */ { + // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it + // is incomplete. + // We check this above. + tor_assert_nonfatal(!ZSTD_isError(retval)); + // Start a new frame if this frame is done + if (retval == 0) + return TOR_COMPRESS_DONE; + // Don't check out_len, it might have some space left if the next output + // chunk is larger than the remaining space + else if (*in_len > 0) + return TOR_COMPRESS_BUFFER_FULL; + else + return TOR_COMPRESS_OK; + } + +#else // HAVE_ZSTD. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + + return TOR_COMPRESS_ERROR; +#endif // HAVE_ZSTD. +} + +/** Deallocate <b>state</b>. */ +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zstd_allocation, state->allocation); + +#ifdef HAVE_ZSTD + if (state->compress) { + ZSTD_freeCStream(state->u.compress_stream); + } else { + ZSTD_freeDStream(state->u.decompress_stream); + } +#endif // HAVE_ZSTD. + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all Zstandard + * states. */ +size_t +tor_zstd_get_total_allocation(void) +{ + return atomic_counter_get(&total_zstd_allocation); +} + +/** Initialize the zstd module */ +void +tor_zstd_init(void) +{ + atomic_counter_init(&total_zstd_allocation); +} + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h new file mode 100644 index 0000000000..d3e65c2f16 --- /dev/null +++ b/src/common/compress_zstd.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.h + * \brief Header for compress_zstd.c + **/ + +#ifndef TOR_COMPRESS_ZSTD_H +#define TOR_COMPRESS_ZSTD_H + +int tor_zstd_method_supported(void); + +const char *tor_zstd_get_version_str(void); + +const char *tor_zstd_get_header_version_str(void); + +/** Internal state for an incremental Zstandard compression/decompression. */ +typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; + +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zstd_compress_free(tor_zstd_compress_state_t *state); + +size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); + +size_t tor_zstd_get_total_allocation(void); + +void tor_zstd_init(void); + +#endif // TOR_COMPRESS_ZSTD_H. + diff --git a/src/common/confline.c b/src/common/confline.c new file mode 100644 index 0000000000..691cbf8c6f --- /dev/null +++ b/src/common/confline.c @@ -0,0 +1,525 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "compat.h" +#include "confline.h" +#include "torlog.h" +#include "util.h" +#include "container.h" + +static int config_get_lines_aux(const char *string, config_line_t **result, + int extended, int allow_include, + int *has_include, int recursion_level, + config_line_t **last); +static smartlist_t *config_get_file_list(const char *path); +static int config_get_included_list(const char *path, int recursion_level, + int extended, config_line_t **list, + config_line_t **list_last); +static int config_process_include(const char *path, int recursion_level, + int extended, config_line_t **list, + config_line_t **list_last); + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * append it to *<b>lst</b>. */ +void +config_line_append(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = NULL; + while (*lst) + lst = &((*lst)->next); + + (*lst) = newline; +} + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * and prepend it to *<b>lst</b> */ +void +config_line_prepend(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = *lst; + *lst = newline; +} + +/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or + * NULL if no such key exists. + * + * (In options parsing, this is for handling commandline-only options only; + * other options should be looked up in the appropriate data structure.) */ +const config_line_t * +config_line_find(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcmp(cl->key, key)) + return cl; + } + return NULL; +} + +/** Auxiliary function that does all the work of config_get_lines. + * <b>recursion_level</b> is the count of how many nested %includes we have. + * Returns the a pointer to the last element of the <b>result</b> in + * <b>last</b>. */ +static int +config_get_lines_aux(const char *string, config_line_t **result, int extended, + int allow_include, int *has_include, int recursion_level, + config_line_t **last) +{ + config_line_t *list = NULL, **next, *list_last = NULL; + char *k, *v; + const char *parse_err; + int include_used = 0; + + if (recursion_level > MAX_INCLUDE_RECURSION_LEVEL) { + log_warn(LD_CONFIG, "Error while parsing configuration: more than %d " + "nested %%includes.", MAX_INCLUDE_RECURSION_LEVEL); + return -1; + } + + next = &list; + do { + k = v = NULL; + string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err); + if (!string) { + log_warn(LD_CONFIG, "Error while parsing configuration: %s", + parse_err?parse_err:"<unknown>"); + config_free_lines(list); + tor_free(k); + tor_free(v); + return -1; + } + if (k && v) { + unsigned command = CONFIG_LINE_NORMAL; + if (extended) { + if (k[0] == '+') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + command = CONFIG_LINE_APPEND; + } else if (k[0] == '/') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + tor_free(v); + v = tor_strdup(""); + command = CONFIG_LINE_CLEAR; + } + } + + if (allow_include && !strcmp(k, "%include")) { + tor_free(k); + include_used = 1; + + config_line_t *include_list; + if (config_process_include(v, recursion_level, extended, &include_list, + &list_last) < 0) { + log_warn(LD_CONFIG, "Error reading included configuration " + "file or directory: \"%s\".", v); + config_free_lines(list); + tor_free(v); + return -1; + } + *next = include_list; + if (list_last) + next = &list_last->next; + tor_free(v); + } else { + /* This list can get long, so we keep a pointer to the end of it + * rather than using config_line_append over and over and getting + * n^2 performance. */ + *next = tor_malloc_zero(sizeof(**next)); + (*next)->key = k; + (*next)->value = v; + (*next)->next = NULL; + (*next)->command = command; + list_last = *next; + next = &((*next)->next); + } + } else { + tor_free(k); + tor_free(v); + } + } while (*string); + + if (last) { + *last = list_last; + } + if (has_include) { + *has_include = include_used; + } + *result = list; + return 0; +} + +/** Helper: parse the config string and strdup into key/value + * strings. Set *result to the list, or NULL if parsing the string + * failed. Set *has_include to 1 if <b>result</b> has values from + * %included files. Return 0 on success, -1 on failure. Warn and ignore any + * misformatted lines. + * + * If <b>extended</b> is set, then treat keys beginning with / and with + as + * indicating "clear" and "append" respectively. */ +int +config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include) +{ + return config_get_lines_aux(string, result, extended, 1, has_include, 1, + NULL); +} + +/** Same as config_get_lines_include but does not allow %include */ +int +config_get_lines(const char *string, config_line_t **result, int extended) +{ + return config_get_lines_aux(string, result, extended, 0, NULL, 1, NULL); +} + +/** Adds a list of configuration files present on <b>path</b> to + * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file, + * only that file will be added to <b>file_list</b>. If it is a directory, + * all paths for files on that directory root (no recursion) except for files + * whose name starts with a dot will be added to <b>file_list</b>. + * Return 0 on success, -1 on failure. Ignores empty files. + */ +static smartlist_t * +config_get_file_list(const char *path) +{ + smartlist_t *file_list = smartlist_new(); + file_status_t file_type = file_status(path); + if (file_type == FN_FILE) { + smartlist_add_strdup(file_list, path); + return file_list; + } else if (file_type == FN_DIR) { + smartlist_t *all_files = tor_listdir(path); + if (!all_files) { + smartlist_free(file_list); + return NULL; + } + smartlist_sort_strings(all_files); + SMARTLIST_FOREACH_BEGIN(all_files, char *, f) { + if (f[0] == '.') { + tor_free(f); + continue; + } + + char *fullname; + tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f); + tor_free(f); + + if (file_status(fullname) != FN_FILE) { + tor_free(fullname); + continue; + } + smartlist_add(file_list, fullname); + } SMARTLIST_FOREACH_END(f); + smartlist_free(all_files); + return file_list; + } else if (file_type == FN_EMPTY) { + return file_list; + } else { + smartlist_free(file_list); + return NULL; + } +} + +/** Creates a list of config lines present on included <b>path</b>. + * Set <b>list</b> to the list and <b>list_last</b> to the last element of + * <b>list</b>. Return 0 on success, -1 on failure. */ +static int +config_get_included_list(const char *path, int recursion_level, int extended, + config_line_t **list, config_line_t **list_last) +{ + char *included_conf = read_file_to_str(path, 0, NULL); + if (!included_conf) { + return -1; + } + + if (config_get_lines_aux(included_conf, list, extended, 1, NULL, + recursion_level+1, list_last) < 0) { + tor_free(included_conf); + return -1; + } + + tor_free(included_conf); + return 0; +} + +/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the + * list of configuration settings obtained and <b>list_last</b> to the last + * element of the same list. Return 0 on success, -1 on failure. */ +static int +config_process_include(const char *path, int recursion_level, int extended, + config_line_t **list, config_line_t **list_last) +{ + config_line_t *ret_list = NULL; + config_line_t **next = &ret_list; +#if 0 + // Disabled -- we already unescape_string() on the result. */ + char *unquoted_path = get_unquoted_path(path); + if (!unquoted_path) { + return -1; + } + + smartlist_t *config_files = config_get_file_list(unquoted_path); + if (!config_files) { + tor_free(unquoted_path); + return -1; + } + tor_free(unquoted_path); +#endif + smartlist_t *config_files = config_get_file_list(path); + if (!config_files) { + return -1; + } + + SMARTLIST_FOREACH_BEGIN(config_files, char *, config_file) { + config_line_t *included_list = NULL; + if (config_get_included_list(config_file, recursion_level, extended, + &included_list, list_last) < 0) { + SMARTLIST_FOREACH(config_files, char *, f, tor_free(f)); + smartlist_free(config_files); + return -1; + } + tor_free(config_file); + + *next = included_list; + if (*list_last) + next = &(*list_last)->next; + + } SMARTLIST_FOREACH_END(config_file); + smartlist_free(config_files); + *list = ret_list; + return 0; +} + +/** + * Free all the configuration lines on the linked list <b>front</b>. + */ +void +config_free_lines(config_line_t *front) +{ + config_line_t *tmp; + + while (front) { + tmp = front; + front = tmp->next; + + tor_free(tmp->key); + tor_free(tmp->value); + tor_free(tmp); + } +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>. */ +config_line_t * +config_lines_dup(const config_line_t *inp) +{ + return config_lines_dup_and_filter(inp, NULL); +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>, + * but only the ones that match <b>key</b>. */ +config_line_t * +config_lines_dup_and_filter(const config_line_t *inp, + const char *key) +{ + config_line_t *result = NULL; + config_line_t **next_out = &result; + while (inp) { + if (key && strcasecmpstart(inp->key, key)) { + inp = inp->next; + continue; + } + *next_out = tor_malloc_zero(sizeof(config_line_t)); + (*next_out)->key = tor_strdup(inp->key); + (*next_out)->value = tor_strdup(inp->value); + inp = inp->next; + next_out = &((*next_out)->next); + } + (*next_out) = NULL; + return result; +} + +/** Return true iff a and b contain identical keys and values in identical + * order. */ +int +config_lines_eq(config_line_t *a, config_line_t *b) +{ + while (a && b) { + if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) + return 0; + a = a->next; + b = b->next; + } + if (a || b) + return 0; + return 1; +} + +/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */ +int +config_count_key(const config_line_t *a, const char *key) +{ + int n = 0; + while (a) { + if (!strcasecmp(a->key, key)) { + ++n; + } + a = a->next; + } + return n; +} + +/** Given a string containing part of a configuration file or similar format, + * advance past comments and whitespace and try to parse a single line. If we + * parse a line successfully, set *<b>key_out</b> to a new string holding the + * key portion and *<b>value_out</b> to a new string holding the value portion + * of the line, and return a pointer to the start of the next line. If we run + * out of data, return a pointer to the end of the string. If we encounter an + * error, return NULL and set *<b>err_out</b> (if provided) to an error + * message. + */ +const char * +parse_config_line_from_str_verbose(const char *line, char **key_out, + char **value_out, + const char **err_out) +{ + /* + See torrc_format.txt for a description of the (silly) format this parses. + */ + const char *key, *val, *cp; + int continuation = 0; + + tor_assert(key_out); + tor_assert(value_out); + + *key_out = *value_out = NULL; + key = val = NULL; + /* Skip until the first keyword. */ + while (1) { + while (TOR_ISSPACE(*line)) + ++line; + if (*line == '#') { + while (*line && *line != '\n') + ++line; + } else { + break; + } + } + + if (!*line) { /* End of string? */ + *key_out = *value_out = NULL; + return line; + } + + /* Skip until the next space or \ followed by newline. */ + key = line; + while (*line && !TOR_ISSPACE(*line) && *line != '#' && + ! (line[0] == '\\' && line[1] == '\n')) + ++line; + *key_out = tor_strndup(key, line-key); + + /* Skip until the value. */ + while (*line == ' ' || *line == '\t') + ++line; + + val = line; + + /* Find the end of the line. */ + if (*line == '\"') { // XXX No continuation handling is done here + if (!(line = unescape_string(line, value_out, NULL))) { + if (err_out) + *err_out = "Invalid escape sequence in quoted string"; + return NULL; + } + while (*line == ' ' || *line == '\t') + ++line; + if (*line == '\r' && *(++line) == '\n') + ++line; + if (*line && *line != '#' && *line != '\n') { + if (err_out) + *err_out = "Excess data after quoted string"; + return NULL; + } + } else { + /* Look for the end of the line. */ + while (*line && *line != '\n' && (*line != '#' || continuation)) { + if (*line == '\\' && line[1] == '\n') { + continuation = 1; + line += 2; + } else if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + if (*line == '\n') + ++line; + } else { + ++line; + } + } + + if (*line == '\n') { + cp = line++; + } else { + cp = line; + } + /* Now back cp up to be the last nonspace character */ + while (cp>val && TOR_ISSPACE(*(cp-1))) + --cp; + + tor_assert(cp >= val); + + /* Now copy out and decode the value. */ + *value_out = tor_strndup(val, cp-val); + if (continuation) { + char *v_out, *v_in; + v_out = v_in = *value_out; + while (*v_in) { + if (*v_in == '#') { + do { + ++v_in; + } while (*v_in && *v_in != '\n'); + if (*v_in == '\n') + ++v_in; + } else if (v_in[0] == '\\' && v_in[1] == '\n') { + v_in += 2; + } else { + *v_out++ = *v_in++; + } + } + *v_out = '\0'; + } + } + + if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + } + while (TOR_ISSPACE(*line)) ++line; + + return line; +} + diff --git a/src/common/confline.h b/src/common/confline.h new file mode 100644 index 0000000000..eb863e8fd8 --- /dev/null +++ b/src/common/confline.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONFLINE_H +#define TOR_CONFLINE_H + +/** Ordinary configuration line. */ +#define CONFIG_LINE_NORMAL 0 +/** Appends to previous configuration for the same option, even if we + * would ordinary replace it. */ +#define CONFIG_LINE_APPEND 1 +/* Removes all previous configuration for an option. */ +#define CONFIG_LINE_CLEAR 2 + +#define MAX_INCLUDE_RECURSION_LEVEL 31 + +/** A linked list of lines in a config file, or elsewhere */ +typedef struct config_line_t { + char *key; + char *value; + struct config_line_t *next; + + /** What special treatment (if any) does this line require? */ + unsigned int command:2; + /** If true, subsequent assignments to this linelist should replace + * it, not extend it. Set only on the first item in a linelist in an + * or_options_t. */ + unsigned int fragile:1; +} config_line_t; + +void config_line_append(config_line_t **lst, + const char *key, const char *val); +void config_line_prepend(config_line_t **lst, + const char *key, const char *val); +config_line_t *config_lines_dup(const config_line_t *inp); +config_line_t *config_lines_dup_and_filter(const config_line_t *inp, + const char *key); +const config_line_t *config_line_find(const config_line_t *lines, + const char *key); +int config_lines_eq(config_line_t *a, config_line_t *b); +int config_count_key(const config_line_t *a, const char *key); +int config_get_lines(const char *string, config_line_t **result, int extended); +int config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include); +void config_free_lines(config_line_t *front); +const char *parse_config_line_from_str_verbose(const char *line, + char **key_out, char **value_out, + const char **err_out); +#endif + diff --git a/src/common/container.c b/src/common/container.c index 1448ab403c..689e7e55e9 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/container.h b/src/common/container.h index 00c3ca81ad..db68d892f5 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONTAINER_H @@ -224,6 +224,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ + (void) var ## _sl_idx; \ } STMT_END /** diff --git a/src/common/crypto.c b/src/common/crypto.c index 7cb3330bde..0fc8474832 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -467,7 +467,7 @@ crypto_new_pk_from_rsa_(RSA *rsa) return env; } -/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a +/** Helper, used by tor-gencert.c. Return the RSA from a * crypto_pk_t. */ RSA * crypto_pk_get_rsa_(crypto_pk_t *env) @@ -479,7 +479,7 @@ crypto_pk_get_rsa_(crypto_pk_t *env) * private is set, include the private-key portion of the key. Return a valid * pointer on success, and NULL on failure. */ MOCK_IMPL(EVP_PKEY *, - crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) +crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) { RSA *key = NULL; EVP_PKEY *pkey = NULL; @@ -516,7 +516,7 @@ crypto_dh_get_dh_(crypto_dh_t *dh) * be set. */ MOCK_IMPL(crypto_pk_t *, - crypto_pk_new,(void)) +crypto_pk_new,(void)) { RSA *rsa; @@ -606,7 +606,7 @@ crypto_cipher_free(crypto_cipher_t *env) * Return 0 on success, -1 on failure. */ MOCK_IMPL(int, - crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) +crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) { tor_assert(env); @@ -3459,3 +3459,15 @@ crypto_global_cleanup(void) /** @} */ +#ifdef USE_DMALLOC +/** Tell the crypto library to use Tor's allocation functions rather than + * calling libc's allocation functions directly. Return 0 on success, -1 + * on failure. */ +int +crypto_use_tor_alloc_functions(void) +{ + int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_); + return r ? 0 : -1; +} +#endif + diff --git a/src/common/crypto.h b/src/common/crypto.h index 42345f80e8..c70d91c262 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -131,6 +131,10 @@ int crypto_early_init(void) ATTR_WUR; int crypto_global_init(int hardwareAccel, const char *accelName, const char *accelPath) ATTR_WUR; +#ifdef USE_DMALLOC +int crypto_use_tor_alloc_functions(void); +#endif + void crypto_thread_cleanup(void); int crypto_global_cleanup(void); diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index 5f328e124c..b99f13a93b 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 4011820949..e7790edac0 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_CURVE25519_H diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index 525d25a3e0..188e18c710 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,8 +32,6 @@ #include "ed25519/ref10/ed25519_ref10.h" #include "ed25519/donna/ed25519_donna_tor.h" -#include <openssl/sha.h> - static void pick_ed25519_impl(void); /** An Ed25519 implementation, as a set of function pointers. */ @@ -442,14 +440,16 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, { const char string[] = "Derive high part of ed25519 key from curve25519 key"; ed25519_public_key_t pubkey_check; - SHA512_CTX ctx; - uint8_t sha512_output[64]; + crypto_digest_t *ctx; + uint8_t sha512_output[DIGEST512_LEN]; memcpy(out->seckey.seckey, inp->seckey.secret_key, 32); - SHA512_Init(&ctx); - SHA512_Update(&ctx, out->seckey.seckey, 32); - SHA512_Update(&ctx, string, sizeof(string)); - SHA512_Final(sha512_output, &ctx); + + ctx = crypto_digest512_new(DIGEST_SHA512); + crypto_digest_add_bytes(ctx, (const char*)out->seckey.seckey, 32); + crypto_digest_add_bytes(ctx, (const char*)string, sizeof(string)); + crypto_digest_get_digest(ctx, (char *)sha512_output, sizeof(sha512_output)); + crypto_digest_free(ctx); memcpy(out->seckey.seckey + 32, sha512_output, 32); ed25519_public_key_generate(&out->pubkey, &out->seckey); diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h index f4a4adad68..77a3313adc 100644 --- a/src/common/crypto_ed25519.h +++ b/src/common/crypto_ed25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_ED25519_H diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index aa2a9d1fb0..1d090a8770 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_format.h b/src/common/crypto_format.h index 86c29d319c..390916cf04 100644 --- a/src/common/crypto_format.h +++ b/src/common/crypto_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_FORMAT_H diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 31e37c007d..db8892e376 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index 5dbd2ad91f..076df815a9 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_s2k.h b/src/common/crypto_s2k.h index 9b186450b1..04212b868a 100644 --- a/src/common/crypto_s2k.h +++ b/src/common/crypto_s2k.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_S2K_H_INCLUDED diff --git a/src/common/di_ops.c b/src/common/di_ops.c index 4ed49e1164..e47998107d 100644 --- a/src/common/di_ops.c +++ b/src/common/di_ops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/di_ops.h b/src/common/di_ops.h index 0a154302bf..e174fcc4e4 100644 --- a/src/common/di_ops.h +++ b/src/common/di_ops.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/handles.h b/src/common/handles.h index 1ee2322579..6d7262ab80 100644 --- a/src/common/handles.h +++ b/src/common/handles.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/include.am b/src/common/include.am index 40c463c9d9..1253888815 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -84,6 +84,7 @@ LIBOR_A_SRC = \ src/common/compat.c \ src/common/compat_threads.c \ src/common/compat_time.c \ + src/common/confline.c \ src/common/container.c \ src/common/log.c \ src/common/memarea.c \ @@ -93,21 +94,31 @@ LIBOR_A_SRC = \ src/common/util_format.c \ src/common/util_process.c \ src/common/sandbox.c \ + src/common/storagedir.c \ src/common/workqueue.c \ $(libor_extra_source) \ $(threads_impl_source) \ $(readpassphrase_source) +if USE_RUST +else +LIBOR_A_SRC += src/common/compat_rust.c +endif + src/common/src_common_libor_testing_a-log.$(OBJEXT) \ src/common/log.$(OBJEXT): micro-revision.i LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress.c \ + src/common/compress_lzma.c \ + src/common/compress_none.c \ + src/common/compress_zlib.c \ + src/common/compress_zstd.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ src/common/crypto_format.c \ - src/common/torgzip.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -141,8 +152,15 @@ COMMONHEADERS = \ src/common/compat.h \ src/common/compat_libevent.h \ src/common/compat_openssl.h \ + src/common/compat_rust.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress.h \ + src/common/compress_lzma.h \ + src/common/compress_none.h \ + src/common/compress_zlib.h \ + src/common/compress_zstd.h \ + src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ src/common/crypto_curve25519.h \ @@ -157,9 +175,9 @@ COMMONHEADERS = \ src/common/procmon.h \ src/common/pubsub.h \ src/common/sandbox.h \ + src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ - src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/log.c b/src/common/log.c index 5f7151bf0c..6a5819064a 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1086,7 +1086,7 @@ add_file_log(const log_severity_list_t *severity, const char *filename, int open_flags = O_WRONLY|O_CREAT; open_flags |= truncate_log ? O_TRUNC : O_APPEND; - fd = tor_open_cloexec(filename, open_flags, 0644); + fd = tor_open_cloexec(filename, open_flags, 0640); if (fd<0) return -1; if (tor_fd_seekend(fd)<0) { @@ -1177,7 +1177,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", NULL + "SCHED", "GUARD", "CONSDIFF", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, diff --git a/src/common/memarea.c b/src/common/memarea.c index 7d16b702e3..659d1edf54 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** \file memarea.c @@ -12,6 +12,9 @@ #include "util.h" #include "compat.h" #include "torlog.h" +#include "container.h" + +#ifndef DISABLE_MEMORY_SENTINELS /** If true, we try to detect any attempts to write beyond the length of a * memarea. */ @@ -304,3 +307,91 @@ memarea_assert_ok(memarea_t *area) } } +#else + +struct memarea_t { + smartlist_t *pieces; +}; + +memarea_t * +memarea_new(void) +{ + memarea_t *ma = tor_malloc_zero(sizeof(memarea_t)); + ma->pieces = smartlist_new(); + return ma; +} +void +memarea_drop_all(memarea_t *area) +{ + memarea_clear(area); + smartlist_free(area->pieces); + tor_free(area); +} +void +memarea_clear(memarea_t *area) +{ + SMARTLIST_FOREACH(area->pieces, void *, p, tor_free_(p)); + smartlist_clear(area->pieces); +} +int +memarea_owns_ptr(const memarea_t *area, const void *ptr) +{ + SMARTLIST_FOREACH(area->pieces, const void *, p, if (ptr == p) return 1;); + return 0; +} + +void * +memarea_alloc(memarea_t *area, size_t sz) +{ + void *result = tor_malloc(sz); + smartlist_add(area->pieces, result); + return result; +} + +void * +memarea_alloc_zero(memarea_t *area, size_t sz) +{ + void *result = tor_malloc_zero(sz); + smartlist_add(area->pieces, result); + return result; +} +void * +memarea_memdup(memarea_t *area, const void *s, size_t n) +{ + void *r = memarea_alloc(area, n); + memcpy(r, s, n); + return r; +} +char * +memarea_strdup(memarea_t *area, const char *s) +{ + size_t n = strlen(s); + char *r = memarea_alloc(area, n+1); + memcpy(r, s, n); + r[n] = 0; + return r; +} +char * +memarea_strndup(memarea_t *area, const char *s, size_t n) +{ + size_t ln = strnlen(s, n); + char *r = memarea_alloc(area, ln+1); + memcpy(r, s, ln); + r[ln] = 0; + return r; +} +void +memarea_get_stats(memarea_t *area, + size_t *allocated_out, size_t *used_out) +{ + (void)area; + *allocated_out = *used_out = 128; +} +void +memarea_assert_ok(memarea_t *area) +{ + (void)area; +} + +#endif + diff --git a/src/common/memarea.h b/src/common/memarea.h index 85bca51ad3..85012c1c34 100644 --- a/src/common/memarea.h +++ b/src/common/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Tor dependencies */ diff --git a/src/common/procmon.c b/src/common/procmon.c index c485c760c7..d49e7f18f5 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/procmon.h b/src/common/procmon.h index 49ead24092..b07cff2c4a 100644 --- a/src/common/procmon.h +++ b/src/common/procmon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.c b/src/common/pubsub.c index b3faf40e00..336e8a6e7f 100644 --- a/src/common/pubsub.c +++ b/src/common/pubsub.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.h b/src/common/pubsub.h index bbb4f02a42..6f4ce08754 100644 --- a/src/common/pubsub.h +++ b/src/common/pubsub.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 8b92558a91..52caa4fcc6 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,8 +19,14 @@ #define _LARGEFILE64_SOURCE #endif -/** Malloc mprotect limit in bytes. */ -#define MALLOC_MP_LIM 1048576 +/** Malloc mprotect limit in bytes. + * + * 28/06/2017: This value was increased from 16 MB to 20 MB after we introduced + * LZMA support in Tor (0.3.1.1-alpha). We limit our LZMA coder to 16 MB, but + * liblzma have a small overhead that we need to compensate for to avoid being + * killed by the sandbox. + */ +#define MALLOC_MP_LIM (20*1024*1024) #include <stdio.h> #include <string.h> diff --git a/src/common/sandbox.h b/src/common/sandbox.h index c5963e3119..a6b83153af 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/storagedir.c b/src/common/storagedir.c new file mode 100644 index 0000000000..31933f64c2 --- /dev/null +++ b/src/common/storagedir.c @@ -0,0 +1,569 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "container.h" +#include "compat.h" +#include "confline.h" +#include "memarea.h" +#include "sandbox.h" +#include "storagedir.h" +#include "torlog.h" +#include "util.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#define FNAME_MIN_NUM 1000 + +/** A storage_dir_t represents a directory full of similar cached + * files. Filenames are decimal integers. Files can be cleaned as needed + * to limit total disk usage. */ +struct storage_dir_t { + /** Directory holding the files for this storagedir. */ + char *directory; + /** Either NULL, or a directory listing of the directory (as a smartlist + * of strings */ + smartlist_t *contents; + /** The largest number of non-temporary files we'll place in the + * directory. */ + int max_files; + /** If true, then 'usage' has been computed. */ + int usage_known; + /** The total number of bytes used in this directory */ + uint64_t usage; +}; + +/** Create or open a new storage directory at <b>dirname</b>, with + * capacity for up to <b>max_files</b> files. + */ +storage_dir_t * +storage_dir_new(const char *dirname, int max_files) +{ + if (check_private_dir(dirname, CPD_CREATE, NULL) < 0) + return NULL; + + storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t)); + d->directory = tor_strdup(dirname); + d->max_files = max_files; + return d; +} + +/** + * Drop all in-RAM storage for <b>d</b>. Does not delete any files. + */ +void +storage_dir_free(storage_dir_t *d) +{ + if (d == NULL) + return; + tor_free(d->directory); + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + tor_free(d); +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that <b>d</b> will need. + * + * The presence of this function is why we need an upper limit on the + * number of files in a storage_dir_t: we need to approve file operations + * one by one. + */ +int +storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg) +{ + int problems = 0; + int idx; + for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) { + char *path = NULL, *tmppath = NULL; + tor_asprintf(&path, "%s/%d", d->directory, idx); + tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx); + + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_rename(cfg, + tor_strdup(tmppath), tor_strdup(path)); + + tor_free(path); + tor_free(tmppath); + } + + return problems ? -1 : 0; +} + +/** + * Remove all files in <b>d</b> whose names end with ".tmp". + * + * Requires that the contents field of <b>d</b> is set. + */ +static void +storage_dir_clean_tmpfiles(storage_dir_t *d) +{ + if (!d->contents) + return; + SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) { + if (strcmpend(fname, ".tmp")) + continue; + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + if (unlink(sandbox_intern_string(path))) { + log_warn(LD_FS, "Unable to unlink %s while cleaning " + "temporary files: %s", escaped(path), strerror(errno)); + tor_free(path); + continue; + } + tor_free(path); + SMARTLIST_DEL_CURRENT(d->contents, fname); + tor_free(fname); + } SMARTLIST_FOREACH_END(fname); + + d->usage_known = 0; +} + +/** + * Re-scan the directory <b>d</b> to learn its contents. + */ +static int +storage_dir_rescan(storage_dir_t *d) +{ + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + d->usage = 0; + d->usage_known = 0; + if (NULL == (d->contents = tor_listdir(d->directory))) { + return -1; + } + storage_dir_clean_tmpfiles(d); + return 0; +} + +/** + * Return a smartlist containing the filenames within <b>d</b>. + */ +const smartlist_t * +storage_dir_list(storage_dir_t *d) +{ + if (! d->contents) + storage_dir_rescan(d); + return d->contents; +} + +/** + * Return the total number of bytes used for storage in <b>d</b>. + */ +uint64_t +storage_dir_get_usage(storage_dir_t *d) +{ + if (d->usage_known) + return d->usage; + + uint64_t total = 0; + SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) { + char *path = NULL; + struct stat st; + tor_asprintf(&path, "%s/%s", d->directory, cp); + if (stat(sandbox_intern_string(path), &st) == 0) { + total += st.st_size; + } + tor_free(path); + } SMARTLIST_FOREACH_END(cp); + + d->usage = total; + d->usage_known = 1; + return d->usage; +} + +/** Mmap a specified file within <b>d</b>. */ +tor_mmap_t * +storage_dir_map(storage_dir_t *d, const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + tor_mmap_t *result = tor_mmap_file(path); + tor_free(path); + return result; +} + +/** Read a file within <b>d</b> into a newly allocated buffer. Set + * *<b>sz_out</b> to its size. */ +uint8_t * +storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out) +{ + const int flags = bin ? RFTS_BIN : 0; + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + struct stat st; + char *contents = read_file_to_str(path, flags, &st); + if (contents && sz_out) { + // it fits in RAM, so we know its size is less than SIZE_MAX +#if UINT64_MAX > SIZE_MAX + tor_assert((uint64_t)st.st_size <= SIZE_MAX); +#endif + *sz_out = (size_t) st.st_size; + } + + tor_free(path); + return (uint8_t *) contents; +} + +/** Helper: Find an unused filename within the directory */ +static char * +find_unused_fname(storage_dir_t *d) +{ + if (!d->contents) { + if (storage_dir_rescan(d) < 0) + return NULL; + } + + char buf[16]; + int i; + /* Yuck; this is quadratic. Fortunately, that shouldn't matter much, + * since disk writes are more expensive by a lot. */ + for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) { + tor_snprintf(buf, sizeof(buf), "%d", i); + if (!smartlist_contains_string(d->contents, buf)) { + return tor_strdup(buf); + } + } + return NULL; +} + +/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of + * sized_chunk_t rather than a single byte array. */ +static int +storage_dir_save_chunks_to_file(storage_dir_t *d, + const smartlist_t *chunks, + int binary, + char **fname_out) +{ + uint64_t total_length = 0; + char *fname = find_unused_fname(d); + if (!fname) + return -1; + + SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch, + total_length += ch->len); + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + + int r = write_chunks_to_file(path, chunks, binary, 0); + if (r == 0) { + if (d->usage_known) + d->usage += total_length; + if (fname_out) { + *fname_out = tor_strdup(fname); + } + if (d->contents) + smartlist_add(d->contents, tor_strdup(fname)); + } + tor_free(fname); + tor_free(path); + return r; +} + +/** Try to write the <b>length</b> bytes at <b>data</b> into a new file + * in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a + * newly allocated string containing the filename. On failure, return + * -1. */ +int +storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out) +{ + smartlist_t *chunks = smartlist_new(); + sized_chunk_t chunk = { (const char *)data, length }; + smartlist_add(chunks, &chunk); + int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out); + smartlist_free(chunks); + return r; +} + +/** + * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string + * <b>str</b>. + */ +int +storage_dir_save_string_to_file(storage_dir_t *d, + const char *str, + int binary, + char **fname_out) +{ + return storage_dir_save_bytes_to_file(d, + (const uint8_t*)str, strlen(str), binary, fname_out); +} + +/** + * As storage_dir_save_bytes_to_file, but associates the data with the + * key-value pairs in <b>labels</b>. Files stored in this format can be + * recovered with storage_dir_map_labeled() or storage_dir_read_labeled(). + */ +int +storage_dir_save_labeled_to_file(storage_dir_t *d, + const config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out) +{ + /* + * The storage format is to prefix the data with the key-value pairs in + * <b>labels</b>, and a single NUL separator. But code outside this module + * MUST NOT rely on that format. + */ + + smartlist_t *chunks = smartlist_new(); + memarea_t *area = memarea_new(); + const config_line_t *line; + for (line = labels; line; line = line->next) { + sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t)); + sz->len = strlen(line->key) + 1 + strlen(line->value) + 1; + const size_t allocated = sz->len + 1; + char *bytes = memarea_alloc(area, allocated); + tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value); + sz->bytes = bytes; + smartlist_add(chunks, sz); + } + + sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t)); + nul->len = 1; + nul->bytes = "\0"; + smartlist_add(chunks, nul); + + sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t)); + datachunk->bytes = (const char *)data; + datachunk->len = length; + smartlist_add(chunks, datachunk); + + int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out); + smartlist_free(chunks); + memarea_drop_all(area); + return r; +} + +/** + * Map a file that was created with storage_dir_save_labeled_to_file(). On + * failure, return NULL. On success, write a set of newly allocated labels + * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and + * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t + * object whose contents should not be used -- it needs to be kept around, + * though, for as long as <b>data_out</b> is going to be valid. + */ +tor_mmap_t * +storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + const uint8_t **data_out, + size_t *sz_out) +{ + tor_mmap_t *m = storage_dir_map(dir, fname); + if (! m) + goto err; + const char *nulp = memchr(m->data, '\0', m->size); + if (! nulp) + goto err; + if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) + goto err; + size_t offset = nulp - m->data + 1; + tor_assert(offset <= m->size); + *data_out = (const uint8_t *)(m->data + offset); + *sz_out = m->size - offset; + + return m; + err: + tor_munmap_file(m); + return NULL; +} + +/** As storage_dir_map_labeled, but return a new byte array containing the + * data. */ +uint8_t * +storage_dir_read_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + size_t *sz_out) +{ + const uint8_t *data = NULL; + tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out, + &data, sz_out); + if (m == NULL) + return NULL; + uint8_t *result = tor_memdup(data, *sz_out); + tor_munmap_file(m); + return result; +} + +/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>. + * This function is a no-op if <b>d->usage_known</b> is 0. */ +static void +storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size) +{ + if (d->usage_known) { + if (! BUG(d->usage < removed_file_size)) { + /* This bug can also be triggered if an external process resized a file + * between the call to storage_dir_get_usage() that last checked + * actual usage (rather than relaying on cached usage), and the call to + * this function. */ + d->usage -= removed_file_size; + } else { + /* If we underflowed the cached directory size, re-check the sizes of all + * the files in the directory. This makes storage_dir_shrink() quadratic, + * but only if a process is continually changing file sizes in the + * storage directory (in which case, we have bigger issues). + * + * We can't just reset usage_known, because storage_dir_shrink() relies + * on knowing the usage. */ + storage_dir_rescan(d); + (void)storage_dir_get_usage(d); + } + } +} + +/** + * Remove the file called <b>fname</b> from <b>d</b>. + */ +void +storage_dir_remove_file(storage_dir_t *d, + const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + const char *ipath = sandbox_intern_string(path); + + uint64_t size = 0; + if (d->usage_known) { + struct stat st; + if (stat(ipath, &st) == 0) { + size = st.st_size; + } + } + if (unlink(ipath) == 0) { + storage_dir_reduce_usage(d, size); + } else { + log_warn(LD_FS, "Unable to unlink %s while removing file: %s", + escaped(path), strerror(errno)); + tor_free(path); + return; + } + if (d->contents) { + smartlist_string_remove(d->contents, fname); + } + + tor_free(path); +} + +/** Helper type: used to sort the members of storage directory by mtime. */ +typedef struct shrinking_dir_entry_t { + time_t mtime; + uint64_t size; + char *path; +} shrinking_dir_entry_t; + +/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */ +static int +shrinking_dir_entry_compare(const void *a_, const void *b_) +{ + const shrinking_dir_entry_t *a = a_; + const shrinking_dir_entry_t *b = b_; + + if (a->mtime < b->mtime) + return -1; + else if (a->mtime > b->mtime) + return 1; + else + return 0; +} + +/** + * Try to free space by removing the oldest files in <b>d</b>. Delete + * until no more than <b>target_size</b> bytes are left, and at least + * <b>min_to_remove</b> files have been removed... or until there is + * nothing left to remove. + * + * Return 0 on success; -1 on failure. + */ +int +storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove) +{ + if (d->usage_known && d->usage <= target_size && !min_to_remove) { + /* Already small enough. */ + return 0; + } + + if (storage_dir_rescan(d) < 0) + return -1; + + const uint64_t orig_usage = storage_dir_get_usage(d); + if (orig_usage <= target_size && !min_to_remove) { + /* Okay, small enough after rescan! */ + return 0; + } + + const int n = smartlist_len(d->contents); + shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t)); + SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) { + shrinking_dir_entry_t *ent = &ents[fname_sl_idx]; + struct stat st; + tor_asprintf(&ent->path, "%s/%s", d->directory, fname); + if (stat(sandbox_intern_string(ent->path), &st) == 0) { + ent->mtime = st.st_mtime; + ent->size = st.st_size; + } + } SMARTLIST_FOREACH_END(fname); + + qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare); + + int idx = 0; + while ((d->usage > target_size || min_to_remove > 0) && idx < n) { + if (unlink(sandbox_intern_string(ents[idx].path)) == 0) { + storage_dir_reduce_usage(d, ents[idx].size); + --min_to_remove; + } + ++idx; + } + + for (idx = 0; idx < n; ++idx) { + tor_free(ents[idx].path); + } + tor_free(ents); + + storage_dir_rescan(d); + + return 0; +} + +/** Remove all files in <b>d</b>. */ +int +storage_dir_remove_all(storage_dir_t *d) +{ + return storage_dir_shrink(d, 0, d->max_files); +} + +/** + * Return the largest number of non-temporary files we're willing to + * store in <b>d</b>. + */ +int +storage_dir_get_max_files(storage_dir_t *d) +{ + return d->max_files; +} + diff --git a/src/common/storagedir.h b/src/common/storagedir.h new file mode 100644 index 0000000000..db25057e65 --- /dev/null +++ b/src/common/storagedir.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_STORAGEDIR_H +#define TOR_STORAGEDIR_H + +typedef struct storage_dir_t storage_dir_t; +struct config_line_t; +struct sandbox_cfg_elem; + +storage_dir_t * storage_dir_new(const char *dirname, int n_files); +void storage_dir_free(storage_dir_t *d); +int storage_dir_register_with_sandbox(storage_dir_t *d, + struct sandbox_cfg_elem **cfg); +const smartlist_t *storage_dir_list(storage_dir_t *d); +uint64_t storage_dir_get_usage(storage_dir_t *d); +tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname); +uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin, + size_t *sz_out); +int storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out); +int storage_dir_save_string_to_file(storage_dir_t *d, + const char *data, + int binary, + char **fname_out); +int storage_dir_save_labeled_to_file(storage_dir_t *d, + const struct config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out); +tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + struct config_line_t **labels_out, + const uint8_t **data_out, + size_t *size_out); +uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname, + struct config_line_t **labels_out, + size_t *sz_out); +void storage_dir_remove_file(storage_dir_t *d, + const char *fname); +int storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove); +int storage_dir_remove_all(storage_dir_t *d); +int storage_dir_get_max_files(storage_dir_t *d); + +#endif + diff --git a/src/common/testsupport.h b/src/common/testsupport.h index 9ad2ba77e0..9fc96199b4 100644 --- a/src/common/testsupport.h +++ b/src/common/testsupport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TESTSUPPORT_H diff --git a/src/common/timers.c b/src/common/timers.c index e1ad47b15b..6f4a6c30f0 100644 --- a/src/common/timers.c +++ b/src/common/timers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/timers.h b/src/common/timers.h index c5246a3335..e816630e6d 100644 --- a/src/common/timers.h +++ b/src/common/timers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TIMERS_H diff --git a/src/common/torgzip.c b/src/common/torgzip.c deleted file mode 100644 index 04ae353cf4..0000000000 --- a/src/common/torgzip.c +++ /dev/null @@ -1,584 +0,0 @@ -/* Copyright (c) 2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.c - * \brief A simple in-memory gzip implementation. - **/ - -#include "orconfig.h" - -#include <stdlib.h> -#include <stdio.h> -#include <assert.h> -#include <string.h> -#include "torint.h" - -#ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> -#endif - -#include "util.h" -#include "torlog.h" -#include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include <zlib.h> - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ -static inline int -method_bits(compress_method_t method, zlib_compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - -static inline int -get_memlevel(zlib_compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - -/** @{ */ -/* These macros define the maximum allowable compression factor. Anything of - * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to - * have an uncompression factor (uncompressed size:compressed size ratio) of - * any greater than MAX_UNCOMPRESSION_FACTOR. - * - * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to - * be small to limit the attack multiplier, but we also want it to be large - * enough so that no legitimate document --even ones we might invent in the - * future -- ever compresses by a factor of greater than - * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably - * large range of possible values. IMO, anything over 8 is probably safe; IMO - * anything under 50 is probably sufficient. - */ -#define MAX_UNCOMPRESSION_FACTOR 25 -#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) -/** @} */ - -/** Return true if uncompressing an input of size <b>in_size</b> to an input - * of size at least <b>size_out</b> looks like a compression bomb. */ -static int -is_compression_bomb(size_t size_in, size_t size_out) -{ - if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) - return 0; - - return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); -} - -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the method described in <b>method</b>. Store the - * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - */ -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#if defined(OpenBSD) - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); - return -1; -} - -/** Given zero or more zlib-compressed or gzip-compressed strings of - * total length - * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated - * buffer, using the method described in <b>method</b>. Store the uncompressed - * string in *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on - * success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a - * failure; otherwise we decompress as much as we can. Warn about truncated - * or corrupt inputs at <b>protocol_warn_level</b>. - */ -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } - return -1; -} - -/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely - * to be compressed or not. If it is, return the likeliest compression method. - * Otherwise, return UNKNOWN_METHOD. - */ -compress_method_t -detect_compression_method(const char *in, size_t in_len) -{ - if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { - return GZIP_METHOD; - } else if (in_len > 2 && (in[0] & 0x0f) == 8 && - (ntohs(get_uint16(in)) % 31) == 0) { - return ZLIB_METHOD; - } else { - return UNKNOWN_METHOD; - } -} - -/** Internal state for an incremental zlib compression/decompression. The - * body of this struct is not exposed. */ -struct tor_zlib_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ - - /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; -}; - -/** Construct and return a tor_zlib_state_t object using <b>method</b>. If - * <b>compress</b>, it's for compression; otherwise it's for - * decompression. */ -tor_zlib_state_t * -tor_zlib_new(int compress_, compress_method_t method, - zlib_compression_level_t compression_level) -{ - tor_zlib_state_t *out; - int bits, memlevel; - - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } - - out = tor_malloc_zero(sizeof(tor_zlib_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - - total_zlib_allocation += out->allocation; - - return out; - - err: - tor_free(out); - return NULL; -} - -/** Compress/decompress some bytes using <b>state</b>. Read up to - * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes - * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, - * we've reached the end of the input. - * - * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. - * Return TOR_ZLIB_OK if we're processed everything from the input. - * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>. - * Return TOR_ZLIB_ERR if the stream is corrupt. - */ -tor_zlib_output_t -tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) -{ - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; - - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } - - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); - - *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; - - if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_ZLIB_ERR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_ZLIB_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_ZLIB_OK; - return TOR_ZLIB_BUF_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_ZLIB_BUF_FULL; - return TOR_ZLIB_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_ZLIB_ERR; - } -} - -/** Deallocate <b>state</b>. */ -void -tor_zlib_free(tor_zlib_state_t *state) -{ - if (!state) - return; - - total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); - - tor_free(state); -} - -/** Return an approximate number of bytes used in RAM to hold a state with - * window bits <b>windowBits</b> and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - -/** Return the approximate number of bytes allocated for <b>state</b>. */ -size_t -tor_zlib_state_size(const tor_zlib_state_t *state) -{ - return state->allocation; -} - -/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; -} - diff --git a/src/common/torgzip.h b/src/common/torgzip.h deleted file mode 100644 index 00f62dcb45..0000000000 --- a/src/common/torgzip.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.h - * \brief Headers for torgzip.h - **/ - -#ifndef TOR_TORGZIP_H -#define TOR_TORGZIP_H - -/** Enumeration of what kind of compression to use. Only ZLIB_METHOD is - * guaranteed to be supported by the compress/uncompress functions here; - * GZIP_METHOD may be supported if we built against zlib version 1.2 or later - * and is_gzip_supported() returns true. */ -typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 -} compress_method_t; - -/** - * Enumeration to define tradeoffs between memory usage and compression level. - * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most - * memory. - **/ -typedef enum { - HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} zlib_compression_level_t; - -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - -int is_gzip_supported(void); - -const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - -compress_method_t detect_compression_method(const char *in, size_t in_len); - -/** Return values from tor_zlib_process; see that function's documentation for - * details. */ -typedef enum { - TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR -} tor_zlib_output_t; -/** Internal state for an incremental zlib compression/decompression. */ -typedef struct tor_zlib_state_t tor_zlib_state_t; -tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - zlib_compression_level_t level); - -tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_zlib_free(tor_zlib_state_t *state); - -size_t tor_zlib_state_size(const tor_zlib_state_t *state); -size_t tor_zlib_get_total_allocation(void); - -#endif - diff --git a/src/common/torint.h b/src/common/torint.h index 58c30f41a8..ee31459e94 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/torlog.h b/src/common/torlog.h index bc957858d9..6e374b1c11 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -101,8 +101,10 @@ #define LD_SCHED (1u<<22) /** Guard nodes */ #define LD_GUARD (1u<<23) +/** Generation and application of consensus diffs. */ +#define LD_CONSDIFF (1u<<24) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 24 +#define N_LOGGING_DOMAINS 25 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ diff --git a/src/common/tortls.c b/src/common/tortls.c index e3e8830b3e..44db3aec58 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,6 +17,7 @@ #include "orconfig.h" #define TORTLS_PRIVATE +#define TORTLS_OPENSSL_PRIVATE #include <assert.h> #ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/ @@ -459,11 +460,11 @@ tor_x509_name_new(const char *cname) * Return a certificate on success, NULL on failure. */ MOCK_IMPL(STATIC X509 *, - tor_tls_create_certificate,(crypto_pk_t *rsa, - crypto_pk_t *rsa_sign, - const char *cname, - const char *cname_sign, - unsigned int cert_lifetime)) +tor_tls_create_certificate,(crypto_pk_t *rsa, + crypto_pk_t *rsa_sign, + const char *cname, + const char *cname_sign, + unsigned int cert_lifetime)) { /* OpenSSL generates self-signed certificates with random 64-bit serial * numbers, so let's do that too. */ @@ -661,7 +662,7 @@ tor_x509_cert_free(tor_x509_cert_t *cert) * Steals a reference to x509_cert. */ MOCK_IMPL(STATIC tor_x509_cert_t *, - tor_x509_cert_new,(X509 *x509_cert)) +tor_x509_cert_new,(X509 *x509_cert)) { tor_x509_cert_t *cert; EVP_PKEY *pkey; @@ -2284,6 +2285,24 @@ check_cert_lifetime_internal(int severity, const X509 *cert, return 0; } +#ifdef TOR_UNIT_TESTS +/* Testing only: return a new x509 cert with the same contents as <b>inp</b>, + but with the expiration time <b>new_expiration_time</b>, signed with + <b>signing_key</b>. */ +STATIC tor_x509_cert_t * +tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key) +{ + X509 *newc = X509_dup(inp->cert); + X509_time_adj(X509_get_notAfter(newc), 0, &new_expiration_time); + EVP_PKEY *pk = crypto_pk_get_evp_pkey_(signing_key, 1); + tor_assert(X509_sign(newc, pk, EVP_sha256())); + EVP_PKEY_free(pk); + return tor_x509_cert_new(newc); +} +#endif + /** Return the number of bytes available for reading from <b>tls</b>. */ int diff --git a/src/common/tortls.h b/src/common/tortls.h index bb7701cc4b..f430aff70b 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_H @@ -63,12 +63,17 @@ typedef enum { } tor_tls_state_t; #define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t) +struct x509_st; +struct ssl_st; +struct ssl_ctx_st; +struct ssl_session_st; + /** Holds a SSL_CTX object and related state used to configure TLS * connections. */ typedef struct tor_tls_context_t { int refcnt; - SSL_CTX *ctx; + struct ssl_ctx_st *ctx; tor_x509_cert_t *my_link_cert; tor_x509_cert_t *my_id_cert; tor_x509_cert_t *my_auth_cert; @@ -78,7 +83,7 @@ typedef struct tor_tls_context_t { /** Structure that we use for a single certificate. */ struct tor_x509_cert_t { - X509 *cert; + struct x509_st *cert; uint8_t *encoded; size_t encoded_len; unsigned pkey_digests_set : 1; @@ -92,7 +97,7 @@ struct tor_x509_cert_t { struct tor_tls_t { uint32_t magic; tor_tls_context_t *context; /** A link to the context object for this tls. */ - SSL *ssl; /**< An OpenSSL SSL object. */ + struct ssl_st *ssl; /**< An OpenSSL SSL object. */ int socket; /**< The underlying file descriptor for this TLS connection. */ char *address; /**< An address to log when describing this connection. */ tor_tls_state_bitfield_t state : 3; /**< The current SSL state, @@ -128,35 +133,45 @@ struct tor_tls_t { STATIC int tor_errno_to_tls_error(int e); STATIC int tor_tls_get_error(tor_tls_t *tls, int r, int extra, const char *doing, int severity, int domain); -STATIC tor_tls_t *tor_tls_get_by_ssl(const SSL *ssl); +STATIC tor_tls_t *tor_tls_get_by_ssl(const struct ssl_st *ssl); STATIC void tor_tls_allocate_tor_tls_object_ex_data_index(void); +#ifdef TORTLS_OPENSSL_PRIVATE STATIC int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx); -STATIC int tor_tls_classify_client_ciphers(const SSL *ssl, +STATIC int tor_tls_classify_client_ciphers(const struct ssl_st *ssl, STACK_OF(SSL_CIPHER) *peer_ciphers); -STATIC int tor_tls_client_is_using_v2_ciphers(const SSL *ssl); +#endif +STATIC int tor_tls_client_is_using_v2_ciphers(const struct ssl_st *ssl); MOCK_DECL(STATIC void, try_to_extract_certs_from_tls, - (int severity, tor_tls_t *tls, X509 **cert_out, X509 **id_cert_out)); + (int severity, tor_tls_t *tls, struct x509_st **cert_out, + struct x509_st **id_cert_out)); #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY -STATIC size_t SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, +STATIC size_t SSL_SESSION_get_master_key(struct ssl_session_st *s, + uint8_t *out, size_t len); #endif -STATIC void tor_tls_debug_state_callback(const SSL *ssl, int type, int val); -STATIC void tor_tls_server_info_callback(const SSL *ssl, int type, int val); -STATIC int tor_tls_session_secret_cb(SSL *ssl, void *secret, +STATIC void tor_tls_debug_state_callback(const struct ssl_st *ssl, + int type, int val); +STATIC void tor_tls_server_info_callback(const struct ssl_st *ssl, + int type, int val); +#ifdef TORTLS_OPENSSL_PRIVATE +STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret, int *secret_len, STACK_OF(SSL_CIPHER) *peer_ciphers, CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher, void *arg); STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher); -MOCK_DECL(STATIC X509*, tor_tls_create_certificate,(crypto_pk_t *rsa, +#endif +MOCK_DECL(STATIC struct x509_st *, tor_tls_create_certificate, + (crypto_pk_t *rsa, crypto_pk_t *rsa_sign, const char *cname, const char *cname_sign, unsigned int cert_lifetime)); STATIC tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, unsigned flags, int is_client); -MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,(X509 *x509_cert)); +MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new, + (struct x509_st *x509_cert)); STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext, crypto_pk_t *identity, unsigned int key_lifetime, @@ -172,6 +187,11 @@ extern tor_tls_context_t *client_tls_context; extern uint16_t v2_cipher_list[]; extern uint64_t total_bytes_written_over_tls; extern uint64_t total_bytes_written_by_tls; + +STATIC tor_x509_cert_t *tor_x509_cert_replace_expiration( + const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key); #endif #endif /* endif TORTLS_PRIVATE */ diff --git a/src/common/util.c b/src/common/util.c index 4942ee3a9e..5b47028097 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1953,7 +1953,7 @@ parse_http_time(const char *date, struct tm *tm) /** Given an <b>interval</b> in seconds, try to write it to the * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form. - * Return 0 on success, -1 on failure. + * Returns a non-negative integer on success, -1 on failure. */ int format_time_interval(char *out, size_t out_len, long interval) @@ -2118,7 +2118,7 @@ read_all(tor_socket_t fd, char *buf, size_t count, int isSocket) return -1; } - while (numread != count) { + while (numread < count) { if (isSocket) result = tor_socket_recv(fd, buf+numread, count-numread, 0); else @@ -3045,135 +3045,39 @@ unescape_string(const char *s, char **result, size_t *size_out) } } -/** Given a string containing part of a configuration file or similar format, - * advance past comments and whitespace and try to parse a single line. If we - * parse a line successfully, set *<b>key_out</b> to a new string holding the - * key portion and *<b>value_out</b> to a new string holding the value portion - * of the line, and return a pointer to the start of the next line. If we run - * out of data, return a pointer to the end of the string. If we encounter an - * error, return NULL and set *<b>err_out</b> (if provided) to an error - * message. - */ -const char * -parse_config_line_from_str_verbose(const char *line, char **key_out, - char **value_out, - const char **err_out) +/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the + * enclosing quotes. Backslashes are not unescaped. Return the unquoted + * <b>path</b> on sucess or 0 if <b>path</b> is not quoted correctly. */ +char * +get_unquoted_path(const char *path) { - /* - See torrc_format.txt for a description of the (silly) format this parses. - */ - const char *key, *val, *cp; - int continuation = 0; - - tor_assert(key_out); - tor_assert(value_out); + size_t len = strlen(path); - *key_out = *value_out = NULL; - key = val = NULL; - /* Skip until the first keyword. */ - while (1) { - while (TOR_ISSPACE(*line)) - ++line; - if (*line == '#') { - while (*line && *line != '\n') - ++line; - } else { - break; - } + if (len == 0) { + return tor_strdup(""); } - if (!*line) { /* End of string? */ - *key_out = *value_out = NULL; - return line; + int has_start_quote = (path[0] == '\"'); + int has_end_quote = (len > 0 && path[len-1] == '\"'); + if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) { + return NULL; } - /* Skip until the next space or \ followed by newline. */ - key = line; - while (*line && !TOR_ISSPACE(*line) && *line != '#' && - ! (line[0] == '\\' && line[1] == '\n')) - ++line; - *key_out = tor_strndup(key, line-key); - - /* Skip until the value. */ - while (*line == ' ' || *line == '\t') - ++line; - - val = line; - - /* Find the end of the line. */ - if (*line == '\"') { // XXX No continuation handling is done here - if (!(line = unescape_string(line, value_out, NULL))) { - if (err_out) - *err_out = "Invalid escape sequence in quoted string"; - return NULL; - } - while (*line == ' ' || *line == '\t') - ++line; - if (*line == '\r' && *(++line) == '\n') - ++line; - if (*line && *line != '#' && *line != '\n') { - if (err_out) - *err_out = "Excess data after quoted string"; + char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1); + char *s = unquoted_path; + size_t i; + for (i = has_start_quote; i < len - has_end_quote; i++) { + if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) { + *(s-1) = path[i]; + } else if (path[i] != '\"') { + *s++ = path[i]; + } else { /* unescaped quote */ + tor_free(unquoted_path); return NULL; } - } else { - /* Look for the end of the line. */ - while (*line && *line != '\n' && (*line != '#' || continuation)) { - if (*line == '\\' && line[1] == '\n') { - continuation = 1; - line += 2; - } else if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); - if (*line == '\n') - ++line; - } else { - ++line; - } - } - - if (*line == '\n') { - cp = line++; - } else { - cp = line; - } - /* Now back cp up to be the last nonspace character */ - while (cp>val && TOR_ISSPACE(*(cp-1))) - --cp; - - tor_assert(cp >= val); - - /* Now copy out and decode the value. */ - *value_out = tor_strndup(val, cp-val); - if (continuation) { - char *v_out, *v_in; - v_out = v_in = *value_out; - while (*v_in) { - if (*v_in == '#') { - do { - ++v_in; - } while (*v_in && *v_in != '\n'); - if (*v_in == '\n') - ++v_in; - } else if (v_in[0] == '\\' && v_in[1] == '\n') { - v_in += 2; - } else { - *v_out++ = *v_in++; - } - } - *v_out = '\0'; - } - } - - if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); } - while (TOR_ISSPACE(*line)) ++line; - - return line; + *s = '\0'; + return unquoted_path; } /** Expand any homedir prefix on <b>filename</b>; return a newly allocated @@ -4175,10 +4079,10 @@ tor_process_get_stdout_pipe(process_handle_t *process_handle) } #else /* DOCDOC tor_process_get_stdout_pipe */ -FILE * +int tor_process_get_stdout_pipe(process_handle_t *process_handle) { - return process_handle->stdout_handle; + return process_handle->stdout_pipe; } #endif @@ -4609,10 +4513,6 @@ tor_spawn_background(const char *const filename, const char **argv, log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " "nonblocking in parent process: %s", strerror(errno)); } - /* Open the buffered IO streams */ - process_handle->stdout_handle = fdopen(process_handle->stdout_pipe, "r"); - process_handle->stderr_handle = fdopen(process_handle->stderr_pipe, "r"); - process_handle->stdin_handle = fdopen(process_handle->stdin_pipe, "r"); *process_handle_out = process_handle; return process_handle->status; @@ -4659,14 +4559,9 @@ tor_process_handle_destroy,(process_handle_t *process_handle, if (process_handle->stdin_pipe) CloseHandle(process_handle->stdin_pipe); #else - if (process_handle->stdout_handle) - fclose(process_handle->stdout_handle); - - if (process_handle->stderr_handle) - fclose(process_handle->stderr_handle); - - if (process_handle->stdin_handle) - fclose(process_handle->stdin_handle); + close(process_handle->stdout_pipe); + close(process_handle->stderr_pipe); + close(process_handle->stdin_pipe); clear_waitpid_callback(process_handle->waitpid_cb); #endif @@ -4952,7 +4847,7 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { + while (numread < count) { /* Check if there is anything to read */ retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); if (!retval) { @@ -4998,19 +4893,19 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, return (ssize_t)numread; } #else -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If +/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If * <b>process</b> is NULL, the function will return immediately if there is * nothing more to read. Otherwise data will be read until end of file, or * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the * file has been reached. */ ssize_t -tor_read_all_handle(FILE *h, char *buf, size_t count, +tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof) { size_t numread = 0; - char *retval; + ssize_t result; if (eof) *eof = 0; @@ -5018,34 +4913,28 @@ tor_read_all_handle(FILE *h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { - /* Use fgets because that is what we use in log_from_pipe() */ - retval = fgets(buf+numread, (int)(count-numread), h); - if (NULL == retval) { - if (feof(h)) { - log_debug(LD_GENERAL, "fgets() reached end of file"); - if (eof) - *eof = 1; + while (numread < count) { + result = read(fd, buf+numread, count-numread); + + if (result == 0) { + log_debug(LD_GENERAL, "read() reached end of file"); + if (eof) + *eof = 1; + break; + } else if (result < 0 && errno == EAGAIN) { + if (process) + continue; + else break; - } else { - if (EAGAIN == errno) { - if (process) - continue; - else - break; - } else { - log_warn(LD_GENERAL, "fgets() from handle failed: %s", - strerror(errno)); - return -1; - } - } + } else if (result < 0) { + log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); + return -1; } - tor_assert(retval != NULL); - tor_assert(strlen(retval) + numread <= count); - numread += strlen(retval); + + numread += result; } - log_debug(LD_GENERAL, "fgets() read %d bytes from handle", (int)numread); + log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); return (ssize_t)numread; } #endif @@ -5059,7 +4948,7 @@ tor_read_all_from_process_stdout(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stdout_handle, buf, count, + return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle, NULL); #endif } @@ -5073,7 +4962,7 @@ tor_read_all_from_process_stderr(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stderr_handle, buf, count, + return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle, NULL); #endif } @@ -5267,11 +5156,10 @@ log_from_handle(HANDLE *pipe, int severity) #else /** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set + * <b>fd</b>. Return NULL on error, and set * <b>stream_status_out</b> appropriately. */ MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (FILE *handle, - enum stream_status *stream_status_out)) +tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) { enum stream_status stream_status; char stdout_buf[400]; @@ -5280,13 +5168,13 @@ tor_get_lines_from_handle, (FILE *handle, while (1) { memset(stdout_buf, 0, sizeof(stdout_buf)); - stream_status = get_string_from_pipe(handle, + stream_status = get_string_from_pipe(fd, stdout_buf, sizeof(stdout_buf) - 1); if (stream_status != IO_STREAM_OKAY) goto done; if (!lines) lines = smartlist_new(); - smartlist_add_strdup(lines, stdout_buf); + smartlist_split_string(lines, stdout_buf, "\n", 0, 0); } done: @@ -5294,20 +5182,20 @@ tor_get_lines_from_handle, (FILE *handle, return lines; } -/** Read from stream, and send lines to log at the specified log level. +/** Read from fd, and send lines to log at the specified log level. * Returns 1 if stream is closed normally, -1 if there is a error reading, and * 0 otherwise. Handles lines from tor-fw-helper and * tor_spawn_background() specially. */ static int -log_from_pipe(FILE *stream, int severity, const char *executable, +log_from_pipe(int fd, int severity, const char *executable, int *child_status) { char buf[256]; enum stream_status r; for (;;) { - r = get_string_from_pipe(stream, buf, sizeof(buf) - 1); + r = get_string_from_pipe(fd, buf, sizeof(buf) - 1); if (r == IO_STREAM_CLOSED) { return 1; @@ -5332,7 +5220,7 @@ log_from_pipe(FILE *stream, int severity, const char *executable, } #endif -/** Reads from <b>stream</b> and stores input in <b>buf_out</b> making +/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making * sure it's below <b>count</b> bytes. * If the string has a trailing newline, we strip it off. * @@ -5348,52 +5236,28 @@ log_from_pipe(FILE *stream, int severity, const char *executable, * IO_STREAM_OKAY: If everything went okay and we got a string * in <b>buf_out</b>. */ enum stream_status -get_string_from_pipe(FILE *stream, char *buf_out, size_t count) +get_string_from_pipe(int fd, char *buf_out, size_t count) { - char *retval; - size_t len; + ssize_t ret; tor_assert(count <= INT_MAX); - retval = fgets(buf_out, (int)count, stream); - - if (!retval) { - if (feof(stream)) { - /* Program has closed stream (probably it exited) */ - /* TODO: check error */ - return IO_STREAM_CLOSED; - } else { - if (EAGAIN == errno) { - /* Nothing more to read, try again next time */ - return IO_STREAM_EAGAIN; - } else { - /* There was a problem, abandon this child process */ - return IO_STREAM_TERM; - } - } - } else { - len = strlen(buf_out); - if (len == 0) { - /* this probably means we got a NUL at the start of the string. */ - return IO_STREAM_EAGAIN; - } + ret = read(fd, buf_out, count); - if (buf_out[len - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[len - 1] = '\0'; - } else { - /* No newline; check whether we overflowed the buffer */ - if (!feof(stream)) - log_info(LD_GENERAL, - "Line from stream was truncated: %s", buf_out); - /* TODO: What to do with this error? */ - } + if (ret == 0) + return IO_STREAM_CLOSED; + else if (ret < 0 && errno == EAGAIN) + return IO_STREAM_EAGAIN; + else if (ret < 0) + return IO_STREAM_TERM; - return IO_STREAM_OKAY; - } + if (buf_out[ret - 1] == '\n') { + /* Remove the trailing newline */ + buf_out[ret - 1] = '\0'; + } else + buf_out[ret] = '\0'; - /* We should never get here */ - return IO_STREAM_TERM; + return IO_STREAM_OKAY; } /** Parse a <b>line</b> from tor-fw-helper and issue an appropriate @@ -5630,7 +5494,7 @@ tor_check_port_forwarding(const char *filename, #ifdef _WIN32 stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO); #else - stderr_status = log_from_pipe(child_handle->stderr_handle, + stderr_status = log_from_pipe(child_handle->stderr_pipe, LOG_INFO, filename, &retval); #endif if (handle_fw_helper_output(filename, child_handle) < 0) { diff --git a/src/common/util.h b/src/common/util.h index 13fcc5142d..d56abcee2e 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -163,9 +163,9 @@ int64_t clamp_double_to_int64(double number); void simplify_fraction64(uint64_t *numer, uint64_t *denom); /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> - * and positive <b>b</b>. Works on integer types only. Not defined if a+b can - * overflow. */ -#define CEIL_DIV(a,b) (((a)+(b)-1)/(b)) + * and positive <b>b</b>. Works on integer types only. Not defined if a+(b-1) + * can overflow. */ +#define CEIL_DIV(a,b) (((a)+((b)-1))/(b)) /* Return <b>v</b> if it's between <b>min</b> and <b>max</b>. Otherwise * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if @@ -322,7 +322,7 @@ enum stream_status { const char *stream_status_to_string(enum stream_status stream_status); -enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count); +enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); MOCK_DECL(int,tor_unlink,(const char *pathname)); @@ -389,9 +389,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out) ATTR_MALLOC; const char *unescape_string(const char *s, char **result, size_t *size_out); -const char *parse_config_line_from_str_verbose(const char *line, - char **key_out, char **value_out, - const char **err_out); +char *get_unquoted_path(const char *path); char *expand_filename(const char *filename); MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); int path_is_relative(const char *filename); @@ -463,9 +461,6 @@ struct process_handle_t { int stdin_pipe; int stdout_pipe; int stderr_pipe; - FILE *stdin_handle; - FILE *stdout_handle; - FILE *stderr_handle; pid_t pid; /** If the process has not given us a SIGCHLD yet, this has the * waitpid_callback_t that gets invoked once it has. Otherwise this @@ -488,7 +483,7 @@ int tor_split_lines(struct smartlist_t *sl, char *buf, int len); ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, const process_handle_t *process); #else -ssize_t tor_read_all_handle(FILE *h, char *buf, size_t count, +ssize_t tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof); #endif @@ -502,7 +497,7 @@ int tor_process_get_pid(process_handle_t *process_handle); #ifdef _WIN32 HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); #else -FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle); +int tor_process_get_stdout_pipe(process_handle_t *process_handle); #endif #ifdef _WIN32 @@ -511,7 +506,7 @@ tor_get_lines_from_handle,(HANDLE *handle, enum stream_status *stream_status)); #else MOCK_DECL(struct smartlist_t *, -tor_get_lines_from_handle,(FILE *handle, +tor_get_lines_from_handle,(int fd, enum stream_status *stream_status)); #endif diff --git a/src/common/util_bug.c b/src/common/util_bug.c index c7bfdefe80..3d990e3700 100644 --- a/src/common/util_bug.c +++ b/src/common/util_bug.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_bug.h b/src/common/util_bug.h index 0695806911..ae7e7a37fd 100644 --- a/src/common/util_bug.h +++ b/src/common/util_bug.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_format.c b/src/common/util_format.c index 7e8ee1b868..1f7b8b03aa 100644 --- a/src/common/util_format.c +++ b/src/common/util_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,13 +22,16 @@ #include <stdlib.h> /* Return the base32 encoded size in bytes using the source length srclen. - * The NUL terminated byte is added as well since every base32 encoding - * requires enough space for it. */ + * + * (WATCH OUT: This API counts the terminating NUL byte, but + * base64_encode_size does not.) + */ size_t base32_encoded_size(size_t srclen) { size_t enclen; - enclen = CEIL_DIV(srclen*8, 5) + 1; + tor_assert(srclen < SIZE_T_CEILING / 8); + enclen = BASE32_NOPAD_BUFSIZE(srclen); tor_assert(enclen < INT_MAX && enclen > srclen); return enclen; } @@ -41,7 +44,6 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) size_t nbits = srclen * 8; size_t bit; - tor_assert(srclen < SIZE_T_CEILING/8); /* We need enough space for the encoded data and the extra NUL byte. */ tor_assert(base32_encoded_size(srclen) <= destlen); tor_assert(destlen < SIZE_T_CEILING); @@ -134,6 +136,9 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) /** Return the Base64 encoded size of <b>srclen</b> bytes of data in * bytes. * + * (WATCH OUT: This API <em>does not</em> count the terminating NUL byte, + * but base32_encoded_size does.) + * * If <b>flags</b>&BASE64_ENCODE_MULTILINE is true, return the size * of the encoded output as multiline output (64 character, `\n' terminated * lines). @@ -142,19 +147,16 @@ size_t base64_encode_size(size_t srclen, int flags) { size_t enclen; + + /* Use INT_MAX for overflow checking because base64_encode() returns int. */ tor_assert(srclen < INT_MAX); + tor_assert(CEIL_DIV(srclen, 3) < INT_MAX / 4); - if (srclen == 0) - return 0; + enclen = BASE64_LEN(srclen); + if (flags & BASE64_ENCODE_MULTILINE) + enclen += CEIL_DIV(enclen, BASE64_OPENSSL_LINELEN); - enclen = ((srclen - 1) / 3) * 4 + 4; - if (flags & BASE64_ENCODE_MULTILINE) { - size_t remainder = enclen % BASE64_OPENSSL_LINELEN; - enclen += enclen / BASE64_OPENSSL_LINELEN; - if (remainder) - enclen++; - } - tor_assert(enclen < INT_MAX && enclen > srclen); + tor_assert(enclen < INT_MAX && (enclen == 0 || enclen > srclen)); return enclen; } @@ -311,39 +313,6 @@ base64_encode_nopad(char *dest, size_t destlen, return (int)(out - dest); } -/** As base64_decode, but do not require any padding on the input */ -int -base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen) -{ - if (srclen > SIZE_T_CEILING - 4) - return -1; - char *buf = tor_malloc(srclen + 4); - memcpy(buf, src, srclen+1); - size_t buflen; - switch (srclen % 4) - { - case 0: - default: - buflen = srclen; - break; - case 1: - tor_free(buf); - return -1; - case 2: - memcpy(buf+srclen, "==", 3); - buflen = srclen + 2; - break; - case 3: - memcpy(buf+srclen, "=", 2); - buflen = srclen + 1; - break; - } - int n = base64_decode((char*)dest, destlen, buf, buflen); - tor_free(buf); - return n; -} - #undef BASE64_OPENSSL_LINELEN /** @{ */ @@ -393,15 +362,9 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) const char *eos = src+srclen; uint32_t n=0; int n_idx=0; - char *dest_orig = dest; + size_t di = 0; - /* Max number of bits == srclen*6. - * Number of bytes required to hold all bits == (srclen*6)/8. - * Yes, we want to round down: anything that hangs over the end of a - * byte is padding. */ - if (!size_mul_check(srclen, 3) || destlen < (srclen*3)/4) - return -1; - if (destlen > SIZE_T_CEILING) + if (destlen > INT_MAX) return -1; /* Make sure we leave no uninitialized data in the destination buffer. */ @@ -429,9 +392,11 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) n = (n<<6) | v; if ((++n_idx) == 4) { /* We've accumulated 24 bits in n. Flush them. */ - *dest++ = (n>>16); - *dest++ = (n>>8) & 0xff; - *dest++ = (n) & 0xff; + if (destlen < 3 || di > destlen - 3) + return -1; + dest[di++] = (n>>16); + dest[di++] = (n>>8) & 0xff; + dest[di++] = (n) & 0xff; n_idx = 0; n = 0; } @@ -449,18 +414,21 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) return -1; case 2: /* 12 leftover bits: The last 4 are padding and the first 8 are data. */ - *dest++ = n >> 4; + if (destlen < 1 || di > destlen - 1) + return -1; + dest[di++] = n >> 4; break; case 3: /* 18 leftover bits: The last 2 are padding and the first 16 are data. */ - *dest++ = n >> 10; - *dest++ = n >> 2; + if (destlen < 2 || di > destlen - 2) + return -1; + dest[di++] = n >> 10; + dest[di++] = n >> 2; } - tor_assert((dest-dest_orig) <= (ssize_t)destlen); - tor_assert((dest-dest_orig) <= INT_MAX); + tor_assert(di <= destlen); - return (int)(dest-dest_orig); + return (int)di; } #undef X #undef SP @@ -476,7 +444,8 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen) const char *end; char *cp; - tor_assert(destlen >= srclen*2+1); + tor_assert(srclen < SIZE_T_CEILING / 2 - 1); + tor_assert(destlen >= BASE16_BUFSIZE(srclen)); tor_assert(destlen < SIZE_T_CEILING); /* Make sure we leave no uninitialized data in the destination buffer. */ diff --git a/src/common/util_format.h b/src/common/util_format.h index 20ac711d10..4af8832bbe 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_UTIL_FORMAT_H @@ -10,6 +10,26 @@ #include "testsupport.h" #include "torint.h" +/** @{ */ +/** These macros don't check for overflow. Use them only for constant inputs + * (like array declarations). The *_LEN macros are the raw encoding lengths + * (without terminating NUL), while the *_BUFSIZE macros count the terminating + * NUL. */ +#define BASE64_LEN(n) (CEIL_DIV((n), 3) * 4) +#define BASE32_LEN(n) (CEIL_DIV((n), 5) * 8) +#define BASE16_LEN(n) ((n) * 2) + +#define BASE64_BUFSIZE(n) (BASE64_LEN(n) + 1) +#define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1) +#define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1) + +#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3)) +#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5)) + +#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1) +#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1) +/** @} */ + #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, @@ -17,8 +37,6 @@ int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); int base64_encode_nopad(char *dest, size_t destlen, const uint8_t *src, size_t srclen); -int base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen); /** Characters that can appear (case-insensitively) in a base32 encoding. */ #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567" diff --git a/src/common/util_process.c b/src/common/util_process.c index abda63720c..9e9679b099 100644 --- a/src/common/util_process.c +++ b/src/common/util_process.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_process.h b/src/common/util_process.h index d38301a354..c3a63498b5 100644 --- a/src/common/util_process.h +++ b/src/common/util_process.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/workqueue.c b/src/common/workqueue.c index e1fb663a2a..ec6e257b4c 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -510,12 +510,13 @@ replyqueue_get_socket(replyqueue_t *rq) void replyqueue_process(replyqueue_t *queue) { - if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { + int r = queue->alert.drain_fn(queue->alert.read_fd); + if (r < 0) { //LCOV_EXCL_START static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, "Failure from drain_fd: %s", - tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); + tor_socket_strerror(-r)); //LCOV_EXCL_STOP } diff --git a/src/common/workqueue.h b/src/common/workqueue.h index 54276767b0..7b483eb7ac 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_WORKQUEUE_H @@ -16,7 +16,7 @@ typedef struct threadpool_s threadpool_t; typedef struct workqueue_entry_s workqueue_entry_t; /** Possible return value from a work function: */ -typedef enum { +typedef enum workqueue_reply_t { WQ_RPL_REPLY = 0, /** indicates success */ WQ_RPL_ERROR = 1, /** indicates fatal error */ WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */ |