diff options
Diffstat (limited to 'src/common')
97 files changed, 11597 insertions, 5212 deletions
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index b8c5dd4fea..a1c819fffa 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \ - crypto_curve25519.obj curve25519-donna.obj +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \ + tortls.obj crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj diff --git a/src/common/address.c b/src/common/address.c index 794345a138..9446675712 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -33,7 +33,7 @@ #include <process.h> #include <windows.h> #include <iphlpapi.h> -#endif +#endif /* defined(_WIN32) */ #include "compat.h" #include "util.h" @@ -159,6 +159,8 @@ tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa, tor_assert(a); tor_assert(sa); + /* This memset is redundant; leaving it in to avoid any future accidents, + however. */ memset(a, 0, sizeof(*a)); if (sa->sa_family == AF_INET) { @@ -196,7 +198,7 @@ tor_sockaddr_to_str(const struct sockaddr *sa) tor_asprintf(&result, "unix:%s", s_un->sun_path); return result; } -#endif +#endif /* defined(HAVE_SYS_UN_H) */ if (sa->sa_family == AF_UNSPEC) return tor_strdup("unspec"); @@ -235,8 +237,8 @@ tor_addr_make_null(tor_addr_t *a, sa_family_t family) * * Return 0 on success, -1 on failure; 1 on transient failure. */ -int -tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) +MOCK_IMPL(int, +tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) { /* Perhaps eventually this should be replaced by a tor_getaddrinfo or * something. @@ -303,7 +305,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) return result; } return (err == EAI_AGAIN) ? 1 : -1; -#else +#else /* !(defined(HAVE_GETADDRINFO)) */ struct hostent *ent; int err; #ifdef HAVE_GETHOSTBYNAME_R_6_ARG @@ -328,7 +330,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) #else err = h_errno; #endif -#endif /* endif HAVE_GETHOSTBYNAME_R_6_ARG. */ +#endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */ if (ent) { if (ent->h_addrtype == AF_INET) { tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); @@ -344,7 +346,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) #else return (err == TRY_AGAIN) ? 1 : -1; #endif -#endif +#endif /* defined(HAVE_GETADDRINFO) */ } } @@ -562,8 +564,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name, * and store the result in the <b>outlen</b>-byte buffer at - * <b>out</b>. Return the number of chars written to <b>out</b>, not - * including the trailing \0, on success. Returns -1 on failure. */ + * <b>out</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_addr_to_PTR_name(char *out, size_t outlen, const tor_addr_t *addr) @@ -905,8 +907,8 @@ tor_addr_is_loopback(const tor_addr_t *addr) return (tor_addr_to_ipv4h(addr) & 0xff000000) == 0x7f000000; case AF_UNSPEC: return 0; - default: /* LCOV_EXCL_START */ + default: tor_fragile_assert(); return 0; /* LCOV_EXCL_STOP */ @@ -1029,8 +1031,10 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src) memcpy(dest->addr.in6_addr.s6_addr, src->addr.in6_addr.s6_addr, 16); case AF_UNSPEC: break; + // LCOV_EXCL_START default: - tor_fragile_assert(); // LCOV_EXCL_LINE + tor_fragile_assert(); + // LCOV_EXCL_STOP } } @@ -1121,7 +1125,7 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, case AF_UNIX: /* HACKHACKHACKHACKHACK: * tor_addr_t doesn't contain a copy of sun_path, so it's not - * possible to comapre this at all. + * possible to compare this at all. * * Since the only time we currently actually should be comparing * 2 AF_UNIX addresses is when dealing with ISO_CLIENTADDR (which @@ -1136,8 +1140,8 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, return 0; else return 1; - default: /* LCOV_EXCL_START */ + default: tor_fragile_assert(); return 0; /* LCOV_EXCL_STOP */ @@ -1195,8 +1199,8 @@ tor_addr_hash(const tor_addr_t *addr) return siphash24g(unspec_hash_input, sizeof(unspec_hash_input)); case AF_INET6: return siphash24g(&addr->addr.in6_addr.s6_addr, 16); - default: /* LCOV_EXCL_START */ + default: tor_fragile_assert(); return 0; /* LCOV_EXCL_STOP */ @@ -1432,7 +1436,7 @@ get_interface_addresses_ifaddrs(int severity, sa_family_t family) return result; } -#endif +#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */ #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST @@ -1526,7 +1530,7 @@ get_interface_addresses_win32(int severity, sa_family_t family) return result; } -#endif +#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */ #ifdef HAVE_IFCONF_TO_SMARTLIST @@ -1539,6 +1543,18 @@ get_interface_addresses_win32(int severity, sa_family_t family) #define _SIZEOF_ADDR_IFREQ sizeof #endif +/* Free ifc->ifc_buf safely. */ +static void +ifconf_free_ifc_buf(struct ifconf *ifc) +{ + /* On macOS, tor_free() takes the address of ifc.ifc_buf, which leads to + * undefined behaviour, because pointer-to-pointers are expected to be + * aligned at 8-bytes, but the ifconf structure is packed. So we use + * raw_free() instead. */ + raw_free(ifc->ifc_buf); + ifc->ifc_buf = NULL; +} + /** Convert <b>*buf</b>, an ifreq structure array of size <b>buflen</b>, * into smartlist of <b>tor_addr_t</b> structures. */ @@ -1625,10 +1641,10 @@ get_interface_addresses_ioctl(int severity, sa_family_t family) done: if (fd >= 0) close(fd); - tor_free(ifc.ifc_buf); + ifconf_free_ifc_buf(&ifc); return result; } -#endif +#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ /** Try to ask our network interfaces what addresses they are bound to. * Return a new smartlist of tor_addr_t on success, and NULL on failure. @@ -1684,7 +1700,7 @@ get_interface_address6_via_udp_socket_hack,(int severity, sa_family_t family, tor_addr_t *addr)) { - struct sockaddr_storage my_addr, target_addr; + struct sockaddr_storage target_addr; int sock=-1, r=-1; socklen_t addr_len; @@ -1727,21 +1743,19 @@ get_interface_address6_via_udp_socket_hack,(int severity, goto err; } - if (tor_getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) { + if (tor_addr_from_getsockname(addr, sock) < 0) { int e = tor_socket_errno(sock); log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s", tor_socket_strerror(e)); goto err; } - if (tor_addr_from_sockaddr(addr, (struct sockaddr*)&my_addr, NULL) == 0) { - if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) { - log_fn(severity, LD_NET, "Address that we determined via UDP socket" - " magic is unsuitable for public comms."); - } else { - r=0; - } - } + if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) { + log_fn(severity, LD_NET, "Address that we determined via UDP socket" + " magic is unsuitable for public comms."); + } else { + r=0; + } err: if (sock >= 0) @@ -1783,14 +1797,14 @@ get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr)) break; } SMARTLIST_FOREACH_END(a); - free_interface_address6_list(addrs); + interface_address6_list_free(addrs); return rv; } /** Free a smartlist of IP addresses returned by get_interface_address6_list. */ void -free_interface_address6_list(smartlist_t *addrs) +interface_address6_list_free_(smartlist_t *addrs) { if (addrs != NULL) { SMARTLIST_FOREACH(addrs, tor_addr_t *, a, tor_free(a)); @@ -1805,11 +1819,12 @@ free_interface_address6_list(smartlist_t *addrs) * An empty smartlist means that there are no addresses of the selected type * matching these criteria. * Returns NULL on failure. - * Use free_interface_address6_list to free the returned list. + * Use interface_address6_list_free to free the returned list. */ -MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, - sa_family_t family, - int include_internal)) +MOCK_IMPL(smartlist_t *, +get_interface_address6_list,(int severity, + sa_family_t family, + int include_internal)) { smartlist_t *addrs; tor_addr_t addr; @@ -2077,7 +2092,8 @@ parse_port_range(const char *port, uint16_t *port_min_out, /** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual), * write it as a string into the <b>buf_len</b>-byte buffer in - * <b>buf</b>. + * <b>buf</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len) @@ -2128,7 +2144,8 @@ get_interface_address,(int severity, uint32_t *addr)) } /** Return true if we can tell that <b>name</b> is a canonical name for the - * loopback address. */ + * loopback address. Return true also for *.local hostnames, which are + * multicast DNS names for hosts on the local network. */ int tor_addr_hostname_is_local(const char *name) { @@ -2149,3 +2166,11 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port) return ap; } +/** Return true iff <a>a</b> and <b>b</b> are the same address and port */ +int +tor_addr_port_eq(const tor_addr_port_t *a, + const tor_addr_port_t *b) +{ + return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port; +} + diff --git a/src/common/address.h b/src/common/address.h index d57abd0d9e..c9d9543dee 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -44,7 +44,7 @@ #endif // TODO win32 specific includes -#endif // ADDRESS_PRIVATE +#endif /* defined(ADDRESS_PRIVATE) */ /** The number of bits from an address to consider while doing a masked * comparison. */ @@ -190,7 +190,8 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) */ #define TOR_ADDR_BUF_LEN 48 -int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); +MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family, + tor_addr_t *addr_out)); char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC; /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6 @@ -205,7 +206,9 @@ const char * fmt_addr32(uint32_t addr); MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr)); -void free_interface_address6_list(smartlist_t * addrs); +void interface_address6_list_free_(smartlist_t * addrs);// XXXX +#define interface_address6_list_free(addrs) \ + FREE_AND_NULL(smartlist_t, interface_address6_list_free_, (addrs)) MOCK_DECL(smartlist_t *,get_interface_address6_list,(int severity, sa_family_t family, int include_internal)); @@ -322,13 +325,8 @@ int addr_mask_get_bits(uint32_t mask); int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len); char *tor_dup_ip(uint32_t addr) ATTR_MALLOC; MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr)); -/** Free a smartlist of IP addresses returned by get_interface_address_list. - */ -static inline void -free_interface_address_list(smartlist_t *addrs) -{ - free_interface_address6_list(addrs); -} +#define interface_address_list_free(lst)\ + interface_address6_list_free(lst) /** Return a smartlist of the IPv4 addresses of all interfaces on the server. * Excludes loopback and multicast addresses. Only includes internal addresses * if include_internal is true. (Note that a relay behind NAT may use an @@ -344,6 +342,8 @@ get_interface_address_list(int severity, int include_internal) } tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port); +int tor_addr_port_eq(const tor_addr_port_t *a, + const tor_addr_port_t *b); #ifdef ADDRESS_PRIVATE MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity, @@ -357,23 +357,23 @@ STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa, sa_family_t family); STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity, sa_family_t family); -#endif +#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */ #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST STATIC smartlist_t *ip_adapter_addresses_to_smartlist( const IP_ADAPTER_ADDRESSES *addresses); STATIC smartlist_t *get_interface_addresses_win32(int severity, sa_family_t family); -#endif +#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */ #ifdef HAVE_IFCONF_TO_SMARTLIST STATIC smartlist_t *ifreq_to_smartlist(char *ifr, size_t buflen); STATIC smartlist_t *get_interface_addresses_ioctl(int severity, sa_family_t family); -#endif +#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ -#endif // ADDRESS_PRIVATE +#endif /* defined(ADDRESS_PRIVATE) */ -#endif +#endif /* !defined(TOR_ADDRESS_H) */ diff --git a/src/common/address_set.c b/src/common/address_set.c index 4924cb65c2..b2f4bb4c95 100644 --- a/src/common/address_set.c +++ b/src/common/address_set.c @@ -15,7 +15,7 @@ #include "address.h" #include "compat.h" #include "container.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "siphash.h" @@ -34,7 +34,7 @@ * independent siphashes rather than messing around with bit-shifts. The * approach here is probably more sound, and we should prefer it if&when we * unify the implementations. - **/ + */ struct address_set_t { /** siphash keys to make N_HASHES independent hashes for each address. */ @@ -63,7 +63,7 @@ address_set_new(int max_addresses_guess) } /** - * Release all storage associated with <b>set</b> + * Release all storage associated with <b>set</b>. */ void address_set_free(address_set_t *set) @@ -107,7 +107,7 @@ address_set_add_ipv4h(address_set_t *set, uint32_t addr) } /** - * Return true if <b>addr</b> if a member of <b>set</b>. (And probably, + * Return true if <b>addr</b> is a member of <b>set</b>. (And probably, * return false if <b>addr</b> is not a member of set.) */ int diff --git a/src/common/address_set.h b/src/common/address_set.h index aedf17fc66..28d29f3fdf 100644 --- a/src/common/address_set.h +++ b/src/common/address_set.h @@ -2,7 +2,7 @@ /* See LICENSE for licensing information */ /** - * \file addressset.h + * \file address_set.h * \brief Types to handle sets of addresses. * * This module was first written on a semi-emergency basis to improve the diff --git a/src/common/aes.c b/src/common/aes.c index 8ab2d2fc6e..0c4f9b9bca 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,8 +16,9 @@ #include <ws2tcpip.h> #endif +#include "compat_openssl.h" #include <openssl/opensslv.h> -#include "crypto.h" +#include "crypto_openssl_mgt.h" #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,0) #error "We require OpenSSL >= 1.0.0" @@ -40,6 +41,7 @@ ENABLE_GCC_WARNING(redundant-decls) #include "util.h" #include "torlog.h" #include "di_ops.h" +#include "crypto_util.h" #ifdef ANDROID /* Android's OpenSSL seems to have removed all of its Engine support. */ @@ -66,11 +68,11 @@ ENABLE_GCC_WARNING(redundant-decls) #elif OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,0,1) && \ (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || \ - defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__)) \ + defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__)) #define USE_EVP_AES_CTR -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,1,0) || ... */ /* We have 2 strategies for getting the AES block cipher: Via OpenSSL's * AES_encrypt function, or via OpenSSL's EVP_EncryptUpdate function. @@ -110,12 +112,16 @@ aes_new_cipher(const uint8_t *key, const uint8_t *iv, int key_bits) return (aes_cnt_cipher_t *) cipher; } void -aes_cipher_free(aes_cnt_cipher_t *cipher_) +aes_cipher_free_(aes_cnt_cipher_t *cipher_) { if (!cipher_) return; EVP_CIPHER_CTX *cipher = (EVP_CIPHER_CTX *) cipher_; +#ifdef OPENSSL_1_1_API + EVP_CIPHER_CTX_reset(cipher); +#else EVP_CIPHER_CTX_cleanup(cipher); +#endif EVP_CIPHER_CTX_free(cipher); } void @@ -142,7 +148,7 @@ evaluate_ctr_for_aes(void) { return 0; } -#else +#else /* !(defined(USE_EVP_AES_CTR)) */ /*======================================================================*/ /* Interface to AES code, and counter implementation */ @@ -163,7 +169,7 @@ struct aes_cnt_cipher { uint32_t counter2; uint32_t counter1; uint32_t counter0; -#endif +#endif /* !defined(WORDS_BIGENDIAN) */ union { /** The counter, in big-endian order, as bytes. */ @@ -212,7 +218,7 @@ evaluate_evp_for_aes(int force_val) log_info(LD_CRYPTO, "No AES engine found; using AES_* functions."); should_use_EVP = 0; } -#endif +#endif /* defined(DISABLE_ENGINES) */ return 0; } @@ -254,7 +260,7 @@ evaluate_ctr_for_aes(void) /* LCOV_EXCL_START */ log_err(LD_CRYPTO, "This OpenSSL has a buggy version of counter mode; " "quitting tor."); - exit(1); + exit(1); // exit ok: openssl is broken. /* LCOV_EXCL_STOP */ } return 0; @@ -312,7 +318,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits) cipher->counter1 = 0; cipher->counter2 = 0; cipher->counter3 = 0; -#endif +#endif /* defined(USING_COUNTER_VARS) */ memset(cipher->ctr_buf.buf, 0, sizeof(cipher->ctr_buf.buf)); @@ -324,7 +330,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits) /** Release storage held by <b>cipher</b> */ void -aes_cipher_free(aes_cnt_cipher_t *cipher) +aes_cipher_free_(aes_cnt_cipher_t *cipher) { if (!cipher) return; @@ -341,7 +347,7 @@ aes_cipher_free(aes_cnt_cipher_t *cipher) STMT_END #else #define UPDATE_CTR_BUF(c, n) -#endif +#endif /* defined(USING_COUNTER_VARS) */ /* Helper function to use EVP with openssl's counter-mode wrapper. */ static void @@ -396,9 +402,9 @@ aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv) cipher->counter2 = ntohl(get_uint32(iv+4)); cipher->counter1 = ntohl(get_uint32(iv+8)); cipher->counter0 = ntohl(get_uint32(iv+12)); -#endif +#endif /* defined(USING_COUNTER_VARS) */ cipher->pos = 0; memcpy(cipher->ctr_buf.buf, iv, 16); } -#endif +#endif /* defined(USE_EVP_AES_CTR) */ diff --git a/src/common/aes.h b/src/common/aes.h index 1cda53f2fa..0b17cd55a4 100644 --- a/src/common/aes.h +++ b/src/common/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ @@ -17,11 +17,13 @@ typedef struct aes_cnt_cipher aes_cnt_cipher_t; aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv, int key_bits); -void aes_cipher_free(aes_cnt_cipher_t *cipher); +void aes_cipher_free_(aes_cnt_cipher_t *cipher); +#define aes_cipher_free(cipher) \ + FREE_AND_NULL(aes_cnt_cipher_t, aes_cipher_free_, (cipher)) void aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len); int evaluate_evp_for_aes(int force_value); int evaluate_ctr_for_aes(void); -#endif +#endif /* !defined(TOR_AES_H) */ diff --git a/src/common/backtrace.c b/src/common/backtrace.c index 81e04e94eb..f2498b2aa6 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -37,7 +37,7 @@ #include <sys/ucontext.h> #elif defined(HAVE_UCONTEXT_H) #include <ucontext.h> -#endif +#endif /* defined(HAVE_CYGWIN_SIGNAL_H) || ... */ #define EXPOSE_CLEAN_BACKTRACE #include "backtrace.h" @@ -76,21 +76,21 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) #ifdef PC_FROM_UCONTEXT #if defined(__linux__) const size_t n = 1; -#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \ +#elif defined(__darwin__) || defined(__APPLE__) || defined(OpenBSD) \ || defined(__FreeBSD__) const size_t n = 2; #else const size_t n = 1; -#endif +#endif /* defined(__linux__) || ... */ if (depth <= n) return; stack[n] = (void*) ctx->PC_FROM_UCONTEXT; -#else +#else /* !(defined(PC_FROM_UCONTEXT)) */ (void) depth; (void) ctx; (void) stack; -#endif +#endif /* defined(PC_FROM_UCONTEXT) */ } /** Log a message <b>msg</b> at <b>severity</b> in <b>domain</b>, and follow @@ -202,7 +202,7 @@ remove_bt_handler(void) { tor_mutex_uninit(&cb_buf_mutex); } -#endif +#endif /* defined(USE_BACKTRACE) */ #ifdef NO_BACKTRACE_IMPL void @@ -221,7 +221,7 @@ static void remove_bt_handler(void) { } -#endif +#endif /* defined(NO_BACKTRACE_IMPL) */ /** Set up code to handle generating error messages on crashes. */ int diff --git a/src/common/backtrace.h b/src/common/backtrace.h index b53fd2c668..3d0ab8a90a 100644 --- a/src/common/backtrace.h +++ b/src/common/backtrace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BACKTRACE_H @@ -15,7 +15,7 @@ void clean_up_backtrace_handler(void); defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx); #endif -#endif +#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */ -#endif +#endif /* !defined(TOR_BACKTRACE_H) */ diff --git a/src/common/buffers.c b/src/common/buffers.c new file mode 100644 index 0000000000..a01add9bef --- /dev/null +++ b/src/common/buffers.c @@ -0,0 +1,1146 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.c + * \brief Implements a generic buffer interface. + * + * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush + * from memory, sockets, file descriptors, TLS connections, or another buf_t. + * Buffers are implemented as linked lists of memory chunks. + * + * All socket-backed and TLS-based connection_t objects have a pair of + * buffers: one for incoming data, and one for outcoming data. These are fed + * and drained from functions in connection.c, trigged by events that are + * monitored in main.c. + **/ + +#define BUFFERS_PRIVATE +#include "orconfig.h" +#include <stddef.h> +#include "buffers.h" +#include "compat.h" +#include "compress.h" +#include "util.h" +#include "torint.h" +#include "torlog.h" +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +//#define PARANOIA + +#ifdef PARANOIA +/** Helper: If PARANOIA is defined, assert that the buffer in local variable + * <b>buf</b> is well-formed. */ +#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END +#else +#define check() STMT_NIL +#endif /* defined(PARANOIA) */ + +/* Implementation notes: + * + * After flirting with memmove, and dallying with ring-buffers, we're finally + * getting up to speed with the 1970s and implementing buffers as a linked + * list of small chunks. Each buffer has such a list; data is removed from + * the head of the list, and added at the tail. The list is singly linked, + * and the buffer keeps a pointer to the head and the tail. + * + * Every chunk, except the tail, contains at least one byte of data. Data in + * each chunk is contiguous. + * + * When you need to treat the first N characters on a buffer as a contiguous + * string, use the buf_pullup function to make them so. Don't do this more + * than necessary. + * + * The major free Unix kernels have handled buffers like this since, like, + * forever. + */ + +/* Chunk manipulation functions */ + +#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0]) + +/* We leave this many NUL bytes at the end of the buffer. */ +#ifdef DISABLE_MEMORY_SENTINELS +#define SENTINEL_LEN 0 +#else +#define SENTINEL_LEN 4 +#endif + +/* Header size plus NUL bytes at the end */ +#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN) + +/** Return the number of bytes needed to allocate a chunk to hold + * <b>memlen</b> bytes. */ +#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen)) +/** Return the number of usable bytes in a chunk allocated with + * malloc(<b>memlen</b>). */ +#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD) + +#define DEBUG_SENTINEL + +#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS) +#define DBG_S(s) s +#else +#define DBG_S(s) (void)0 +#endif + +#ifdef DISABLE_MEMORY_SENTINELS +#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL +#else +#define CHUNK_SET_SENTINEL(chunk, alloclen) do { \ + uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen]; \ + DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]); \ + DBG_S(tor_assert(a == b)); \ + memset(a,0,SENTINEL_LEN); \ + } while (0) +#endif /* defined(DISABLE_MEMORY_SENTINELS) */ + +/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem, + * to free up space at the end. */ +static inline void +chunk_repack(chunk_t *chunk) +{ + if (chunk->datalen && chunk->data != &chunk->mem[0]) { + memmove(chunk->mem, chunk->data, chunk->datalen); + } + chunk->data = &chunk->mem[0]; +} + +/** Keep track of total size of allocated chunks for consistency asserts */ +static size_t total_bytes_allocated_in_chunks = 0; +static void +buf_chunk_free_unchecked(chunk_t *chunk) +{ + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + tor_free(chunk); +} +static inline chunk_t * +chunk_new_with_alloc_size(size_t alloc) +{ + chunk_t *ch; + ch = tor_malloc(alloc); + ch->next = NULL; + ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif + ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; + ch->data = &ch->mem[0]; + CHUNK_SET_SENTINEL(ch, alloc); + return ch; +} + +/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a + * new pointer to <b>chunk</b>. Old pointers are no longer valid. */ +static inline chunk_t * +chunk_grow(chunk_t *chunk, size_t sz) +{ + off_t offset; + const size_t memlen_orig = chunk->memlen; + const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); + const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); + tor_assert(sz > chunk->memlen); + offset = chunk->data - chunk->mem; + chunk = tor_realloc(chunk, new_alloc); + chunk->memlen = sz; + chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == orig_alloc); + chunk->DBG_alloc = new_alloc; +#endif + total_bytes_allocated_in_chunks += new_alloc - orig_alloc; + CHUNK_SET_SENTINEL(chunk, new_alloc); + return chunk; +} + +/** Every chunk should take up at least this many bytes. */ +#define MIN_CHUNK_ALLOC 256 +/** No chunk should take up more than this many bytes. */ +#define MAX_CHUNK_ALLOC 65536 + +/** Return the allocation size we'd like to use to hold <b>target</b> + * bytes. */ +size_t +buf_preferred_chunk_size(size_t target) +{ + tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD); + if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC) + return CHUNK_ALLOC_SIZE(target); + size_t sz = MIN_CHUNK_ALLOC; + while (CHUNK_SIZE_WITH_ALLOC(sz) < target) { + sz <<= 1; + } + return sz; +} + +/** Collapse data from the first N chunks from <b>buf</b> into buf->head, + * growing it as necessary, until buf->head has the first <b>bytes</b> bytes + * of data from the buffer, or until buf->head has all the data in <b>buf</b>. + * + * Set *<b>head_out</b> to point to the first byte of available data, and + * *<b>len_out</b> to the number of bytes of data available at + * *<b>head_out</b>. Note that *<b>len_out</b> may be more or less than + * <b>bytes</b>, depending on the number of bytes available. + */ +void +buf_pullup(buf_t *buf, size_t bytes, const char **head_out, size_t *len_out) +{ + chunk_t *dest, *src; + size_t capacity; + if (!buf->head) { + *head_out = NULL; + *len_out = 0; + return; + } + + check(); + if (buf->datalen < bytes) + bytes = buf->datalen; + + capacity = bytes; + if (buf->head->datalen >= bytes) { + *head_out = buf->head->data; + *len_out = buf->head->datalen; + return; + } + + if (buf->head->memlen >= capacity) { + /* We don't need to grow the first chunk, but we might need to repack it.*/ + size_t needed = capacity - buf->head->datalen; + if (CHUNK_REMAINING_CAPACITY(buf->head) < needed) + chunk_repack(buf->head); + tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed); + } else { + chunk_t *newhead; + size_t newsize; + /* We need to grow the chunk. */ + chunk_repack(buf->head); + newsize = CHUNK_SIZE_WITH_ALLOC(buf_preferred_chunk_size(capacity)); + newhead = chunk_grow(buf->head, newsize); + tor_assert(newhead->memlen >= capacity); + if (newhead != buf->head) { + if (buf->tail == buf->head) + buf->tail = newhead; + buf->head = newhead; + } + } + + dest = buf->head; + while (dest->datalen < bytes) { + size_t n = bytes - dest->datalen; + src = dest->next; + tor_assert(src); + if (n >= src->datalen) { + memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen); + dest->datalen += src->datalen; + dest->next = src->next; + if (buf->tail == src) + buf->tail = dest; + buf_chunk_free_unchecked(src); + } else { + memcpy(CHUNK_WRITE_PTR(dest), src->data, n); + dest->datalen += n; + src->data += n; + src->datalen -= n; + tor_assert(dest->datalen == bytes); + } + } + + check(); + *head_out = buf->head->data; + *len_out = buf->head->datalen; +} + +#ifdef TOR_UNIT_TESTS +/* Write sz bytes from cp into a newly allocated buffer buf. + * Returns NULL when passed a NULL cp or zero sz. + * Asserts on failure: only for use in unit tests. + * buf must be freed using buf_free(). */ +buf_t * +buf_new_with_data(const char *cp, size_t sz) +{ + /* Validate arguments */ + if (!cp || sz <= 0) { + return NULL; + } + + tor_assert(sz < SSIZE_T_CEILING); + + /* Allocate a buffer */ + buf_t *buf = buf_new_with_capacity(sz); + tor_assert(buf); + buf_assert_ok(buf); + tor_assert(!buf->head); + + /* Allocate a chunk that is sz bytes long */ + buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz)); + buf->tail = buf->head; + tor_assert(buf->head); + buf_assert_ok(buf); + tor_assert(buf_allocation(buf) >= sz); + + /* Copy the data and size the buffers */ + tor_assert(sz <= buf_slack(buf)); + tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head)); + memcpy(&buf->head->mem[0], cp, sz); + buf->datalen = sz; + buf->head->datalen = sz; + buf->head->data = &buf->head->mem[0]; + buf_assert_ok(buf); + + /* Make sure everything is large enough */ + tor_assert(buf_allocation(buf) >= sz); + tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf)); + /* Does the buffer implementation allocate more than the requested size? + * (for example, by rounding up). If so, these checks will fail. */ + tor_assert(buf_datalen(buf) == sz); + tor_assert(buf_slack(buf) == 0); + + return buf; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** Remove the first <b>n</b> bytes from buf. */ +void +buf_drain(buf_t *buf, size_t n) +{ + tor_assert(buf->datalen >= n); + while (n) { + tor_assert(buf->head); + if (buf->head->datalen > n) { + buf->head->datalen -= n; + buf->head->data += n; + buf->datalen -= n; + return; + } else { + chunk_t *victim = buf->head; + n -= victim->datalen; + buf->datalen -= victim->datalen; + buf->head = victim->next; + if (buf->tail == victim) + buf->tail = NULL; + buf_chunk_free_unchecked(victim); + } + } + check(); +} + +/** Create and return a new buf with default chunk capacity <b>size</b>. + */ +buf_t * +buf_new_with_capacity(size_t size) +{ + buf_t *b = buf_new(); + b->default_chunk_size = buf_preferred_chunk_size(size); + return b; +} + +/** Allocate and return a new buffer with default capacity. */ +buf_t * +buf_new(void) +{ + buf_t *buf = tor_malloc_zero(sizeof(buf_t)); + buf->magic = BUFFER_MAGIC; + buf->default_chunk_size = 4096; + return buf; +} + +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + +/** Remove all data from <b>buf</b>. */ +void +buf_clear(buf_t *buf) +{ + chunk_t *chunk, *next; + buf->datalen = 0; + for (chunk = buf->head; chunk; chunk = next) { + next = chunk->next; + buf_chunk_free_unchecked(chunk); + } + buf->head = buf->tail = NULL; +} + +/** Return the number of bytes stored in <b>buf</b> */ +MOCK_IMPL(size_t, +buf_datalen, (const buf_t *buf)) +{ + return buf->datalen; +} + +/** Return the total length of all chunks used in <b>buf</b>. */ +size_t +buf_allocation(const buf_t *buf) +{ + size_t total = 0; + const chunk_t *chunk; + for (chunk = buf->head; chunk; chunk = chunk->next) { + total += CHUNK_ALLOC_SIZE(chunk->memlen); + } + return total; +} + +/** Return the number of bytes that can be added to <b>buf</b> without + * performing any additional allocation. */ +size_t +buf_slack(const buf_t *buf) +{ + if (!buf->tail) + return 0; + else + return CHUNK_REMAINING_CAPACITY(buf->tail); +} + +/** Release storage held by <b>buf</b>. */ +void +buf_free_(buf_t *buf) +{ + if (!buf) + return; + + buf_clear(buf); + buf->magic = 0xdeadbeef; + tor_free(buf); +} + +/** Return a new copy of <b>in_chunk</b> */ +static chunk_t * +chunk_copy(const chunk_t *in_chunk) +{ + chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif + newch->next = NULL; + if (in_chunk->data) { + off_t offset = in_chunk->data - in_chunk->mem; + newch->data = newch->mem + offset; + } + return newch; +} + +/** Return a new copy of <b>buf</b> */ +buf_t * +buf_copy(const buf_t *buf) +{ + chunk_t *ch; + buf_t *out = buf_new(); + out->default_chunk_size = buf->default_chunk_size; + for (ch = buf->head; ch; ch = ch->next) { + chunk_t *newch = chunk_copy(ch); + if (out->tail) { + out->tail->next = newch; + out->tail = newch; + } else { + out->head = out->tail = newch; + } + } + out->datalen = buf->datalen; + return out; +} + +/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to + * the tail of <b>buf</b>. If <b>capped</b>, don't allocate a chunk bigger + * than MAX_CHUNK_ALLOC. */ +chunk_t * +buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) +{ + chunk_t *chunk; + + if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { + chunk = chunk_new_with_alloc_size(buf->default_chunk_size); + } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { + chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC); + } else { + chunk = chunk_new_with_alloc_size(buf_preferred_chunk_size(capacity)); + } + + chunk->inserted_time = monotime_coarse_get_stamp(); + + if (buf->tail) { + tor_assert(buf->head); + buf->tail->next = chunk; + buf->tail = chunk; + } else { + tor_assert(!buf->head); + buf->head = buf->tail = chunk; + } + check(); + return chunk; +} + +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * timestamp units. Requires the current monotonic timestamp as its + * input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + +/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into + * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set + * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, + * and the number of bytes read otherwise. */ +static inline int +read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, + int *reached_eof, int *socket_error) +{ + ssize_t read_result; + if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) + at_most = CHUNK_REMAINING_CAPACITY(chunk); + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (read_result < 0) { + int e = tor_socket_errno(fd); + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ +#ifdef _WIN32 + if (e == WSAENOBUFS) + log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); +#endif + *socket_error = e; + return -1; + } + return 0; /* would block. */ + } else if (read_result == 0) { + log_debug(LD_NET,"Encountered eof on fd %d", (int)fd); + *reached_eof = 1; + return 0; + } else { /* actually got bytes. */ + buf->datalen += read_result; + chunk->datalen += read_result; + log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result, + (int)buf->datalen); + tor_assert(read_result < INT_MAX); + return (int)read_result; + } +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +/* XXXX indicate "read blocked" somehow? */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + /* XXXX It's stupid to overload the return values for these functions: + * "error status" and "number of bytes read" are not mutually exclusive. + */ + int r = 0; + size_t total_read = 0; + + check(); + tor_assert(reached_eof); + tor_assert(SOCKET_OK(s)); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - at_most)) + return -1; + + while (at_most > total_read) { + size_t readlen = at_most - total_read; + chunk_t *chunk; + if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { + chunk = buf_add_chunk_with_capacity(buf, at_most, 1); + if (readlen > chunk->memlen) + readlen = chunk->memlen; + } else { + size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); + chunk = buf->tail; + if (cap < readlen) + readlen = cap; + } + + r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + check(); + if (r < 0) + return r; /* Error */ + tor_assert(total_read+r < INT_MAX); + total_read += r; + if ((size_t)r < readlen) { /* eof, block, or no more to read. */ + break; + } + } + return (int)total_read; +} + +/** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk + * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct + * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes + * written on success, 0 on blocking, -1 on failure. + */ +static inline int +flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen) +{ + ssize_t write_result; + + if (sz > chunk->datalen) + sz = chunk->datalen; + write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (write_result < 0) { + int e = tor_socket_errno(s); + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ +#ifdef _WIN32 + if (e == WSAENOBUFS) + log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?"); +#endif + return -1; + } + log_debug(LD_NET,"write() would block, returning."); + return 0; + } else { + *buf_flushlen -= write_result; + buf_drain(buf, write_result); + tor_assert(write_result < INT_MAX); + return (int)write_result; + } +} + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + /* XXXX It's stupid to overload the return values for these functions: + * "error status" and "number of bytes flushed" are not mutually exclusive. + */ + int r; + size_t flushed = 0; + tor_assert(buf_flushlen); + tor_assert(SOCKET_OK(s)); + if (BUG(*buf_flushlen > buf->datalen)) { + *buf_flushlen = buf->datalen; + } + if (BUG(sz > *buf_flushlen)) { + sz = *buf_flushlen; + } + + check(); + while (sz) { + size_t flushlen0; + tor_assert(buf->head); + if (buf->head->datalen >= sz) + flushlen0 = sz; + else + flushlen0 = buf->head->datalen; + + r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + check(); + if (r < 0) + return r; + flushed += r; + sz -= r; + if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */ + break; + } + tor_assert(flushed < INT_MAX); + return (int)flushed; +} + +/** Append <b>string_len</b> bytes from <b>string</b> to the end of + * <b>buf</b>. + * + * Return the new length of the buffer on success, -1 on failure. + */ +int +buf_add(buf_t *buf, const char *string, size_t string_len) +{ + if (!string_len) + return (int)buf->datalen; + check(); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - string_len)) + return -1; + + while (string_len) { + size_t copy; + if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail)) + buf_add_chunk_with_capacity(buf, string_len, 1); + + copy = CHUNK_REMAINING_CAPACITY(buf->tail); + if (copy > string_len) + copy = string_len; + memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy); + string_len -= copy; + string += copy; + buf->datalen += copy; + buf->tail->datalen += copy; + } + + check(); + tor_assert(buf->datalen < INT_MAX); + return (int)buf->datalen; +} + +/** Add a nul-terminated <b>string</b> to <b>buf</b>, not including the + * terminating NUL. */ +void +buf_add_string(buf_t *buf, const char *string) +{ + buf_add(buf, string, strlen(string)); +} + +/** As tor_snprintf, but write the results into a buf_t */ +void +buf_add_printf(buf_t *buf, const char *format, ...) +{ + va_list ap; + va_start(ap,format); + buf_add_vprintf(buf, format, ap); + va_end(ap); +} + +/** As tor_vsnprintf, but write the results into a buf_t. */ +void +buf_add_vprintf(buf_t *buf, const char *format, va_list args) +{ + /* XXXX Faster implementations are easy enough, but let's optimize later */ + char *tmp; + tor_vasprintf(&tmp, format, args); + buf_add(buf, tmp, strlen(tmp)); + tor_free(tmp); +} + +/** Return a heap-allocated string containing the contents of <b>buf</b>, plus + * a NUL byte. If <b>sz_out</b> is provided, set *<b>sz_out</b> to the length + * of the returned string, not including the terminating NUL. */ +char * +buf_extract(buf_t *buf, size_t *sz_out) +{ + tor_assert(buf); + + size_t sz = buf_datalen(buf); + char *result; + result = tor_malloc(sz+1); + buf_peek(buf, result, sz); + result[sz] = 0; + if (sz_out) + *sz_out = sz; + return result; +} + +/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b> + * onto <b>string</b>. + */ +void +buf_peek(const buf_t *buf, char *string, size_t string_len) +{ + chunk_t *chunk; + + tor_assert(string); + /* make sure we don't ask for too much */ + tor_assert(string_len <= buf->datalen); + /* buf_assert_ok(buf); */ + + chunk = buf->head; + while (string_len) { + size_t copy = string_len; + tor_assert(chunk); + if (chunk->datalen < copy) + copy = chunk->datalen; + memcpy(string, chunk->data, copy); + string_len -= copy; + string += copy; + chunk = chunk->next; + } +} + +/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store + * them into <b>string</b>. Return the new buffer size. <b>string_len</b> + * must be \<= the number of bytes on the buffer. + */ +int +buf_get_bytes(buf_t *buf, char *string, size_t string_len) +{ + /* There must be string_len bytes in buf; write them onto string, + * then memmove buf back (that is, remove them from buf). + * + * Return the number of bytes still on the buffer. */ + + check(); + buf_peek(buf, string, string_len); + buf_drain(buf, string_len); + check(); + tor_assert(buf->datalen < INT_MAX); + return (int)buf->datalen; +} + +/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to + * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately. + * Return the number of bytes actually copied. + */ +int +buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen) +{ + /* We can do way better here, but this doesn't turn up in any profiles. */ + char b[4096]; + size_t cp, len; + + if (BUG(buf_out->datalen >= INT_MAX)) + return -1; + if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen)) + return -1; + + len = *buf_flushlen; + if (len > buf_in->datalen) + len = buf_in->datalen; + + cp = len; /* Remember the number of bytes we intend to copy. */ + tor_assert(cp < INT_MAX); + while (len) { + /* This isn't the most efficient implementation one could imagine, since + * it does two copies instead of 1, but I kinda doubt that this will be + * critical path. */ + size_t n = len > sizeof(b) ? sizeof(b) : len; + buf_get_bytes(buf_in, b, n); + buf_add(buf_out, b, n); + len -= n; + } + *buf_flushlen -= cp; + return (int)cp; +} + +/** Moves all data from <b>buf_in</b> to <b>buf_out</b>, without copying. + */ +void +buf_move_all(buf_t *buf_out, buf_t *buf_in) +{ + tor_assert(buf_out); + if (!buf_in) + return; + + if (buf_out->head == NULL) { + buf_out->head = buf_in->head; + buf_out->tail = buf_in->tail; + } else { + buf_out->tail->next = buf_in->head; + buf_out->tail = buf_in->tail; + } + + buf_out->datalen += buf_in->datalen; + buf_in->head = buf_in->tail = NULL; + buf_in->datalen = 0; +} + +/** Internal structure: represents a position in a buffer. */ +typedef struct buf_pos_t { + const chunk_t *chunk; /**< Which chunk are we pointing to? */ + int pos;/**< Which character inside the chunk's data are we pointing to? */ + size_t chunk_pos; /**< Total length of all previous chunks. */ +} buf_pos_t; + +/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/ +static void +buf_pos_init(const buf_t *buf, buf_pos_t *out) +{ + out->chunk = buf->head; + out->pos = 0; + out->chunk_pos = 0; +} + +/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current + * position of <b>out</b>, or later. Return -1 if no instances are found; + * otherwise returns the absolute position of the character. */ +static off_t +buf_find_pos_of_char(char ch, buf_pos_t *out) +{ + const chunk_t *chunk; + int pos; + tor_assert(out); + if (out->chunk) { + if (out->chunk->datalen) { + tor_assert(out->pos < (off_t)out->chunk->datalen); + } else { + tor_assert(out->pos == 0); + } + } + pos = out->pos; + for (chunk = out->chunk; chunk; chunk = chunk->next) { + char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos); + if (cp) { + out->chunk = chunk; + tor_assert(cp - chunk->data < INT_MAX); + out->pos = (int)(cp - chunk->data); + return out->chunk_pos + out->pos; + } else { + out->chunk_pos += chunk->datalen; + pos = 0; + } + } + return -1; +} + +/** Advance <b>pos</b> by a single character, if there are any more characters + * in the buffer. Returns 0 on success, -1 on failure. */ +static inline int +buf_pos_inc(buf_pos_t *pos) +{ + ++pos->pos; + if (pos->pos == (off_t)pos->chunk->datalen) { + if (!pos->chunk->next) + return -1; + pos->chunk_pos += pos->chunk->datalen; + pos->chunk = pos->chunk->next; + pos->pos = 0; + } + return 0; +} + +/** Return true iff the <b>n</b>-character string in <b>s</b> appears + * (verbatim) at <b>pos</b>. */ +static int +buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n) +{ + buf_pos_t p; + if (!n) + return 1; + + memcpy(&p, pos, sizeof(p)); + + while (1) { + char ch = p.chunk->data[p.pos]; + if (ch != *s) + return 0; + ++s; + /* If we're out of characters that don't match, we match. Check this + * _before_ we test incrementing pos, in case we're at the end of the + * string. */ + if (--n == 0) + return 1; + if (buf_pos_inc(&p)<0) + return 0; + } +} + +/** Return the first position in <b>buf</b> at which the <b>n</b>-character + * string <b>s</b> occurs, or -1 if it does not occur. */ +int +buf_find_string_offset(const buf_t *buf, const char *s, size_t n) +{ + buf_pos_t pos; + buf_pos_init(buf, &pos); + while (buf_find_pos_of_char(*s, &pos) >= 0) { + if (buf_matches_at_pos(&pos, s, n)) { + tor_assert(pos.chunk_pos + pos.pos < INT_MAX); + return (int)(pos.chunk_pos + pos.pos); + } else { + if (buf_pos_inc(&pos)<0) + return -1; + } + } + return -1; +} + +/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null + * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */ +int +buf_peek_startswith(const buf_t *buf, const char *cmd) +{ + char tmp[PEEK_BUF_STARTSWITH_MAX]; + size_t clen = strlen(cmd); + if (clen == 0) + return 1; + if (BUG(clen > sizeof(tmp))) + return 0; + if (buf->datalen < clen) + return 0; + buf_peek(buf, tmp, clen); + return fast_memeq(tmp, cmd, clen); +} + +/** Return the index within <b>buf</b> at which <b>ch</b> first appears, + * or -1 if <b>ch</b> does not appear on buf. */ +static off_t +buf_find_offset_of_char(buf_t *buf, char ch) +{ + chunk_t *chunk; + off_t offset = 0; + for (chunk = buf->head; chunk; chunk = chunk->next) { + char *cp = memchr(chunk->data, ch, chunk->datalen); + if (cp) + return offset + (cp - chunk->data); + else + offset += chunk->datalen; + } + return -1; +} + +/** Try to read a single LF-terminated line from <b>buf</b>, and write it + * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer + * at <b>data_out</b>. Set *<b>data_len</b> to the number of bytes in the + * line, not counting the terminating NUL. Return 1 if we read a whole line, + * return 0 if we don't have a whole line yet, and return -1 if the line + * length exceeds *<b>data_len</b>. + */ +int +buf_get_line(buf_t *buf, char *data_out, size_t *data_len) +{ + size_t sz; + off_t offset; + + if (!buf->head) + return 0; + + offset = buf_find_offset_of_char(buf, '\n'); + if (offset < 0) + return 0; + sz = (size_t) offset; + if (sz+2 > *data_len) { + *data_len = sz + 2; + return -1; + } + buf_get_bytes(buf, data_out, sz+1); + data_out[sz+1] = '\0'; + *data_len = sz+1; + return 1; +} + +/** Compress or uncompress the <b>data_len</b> bytes in <b>data</b> using the + * compression state <b>state</b>, appending the result to <b>buf</b>. If + * <b>done</b> is true, flush the data in the state and finish the + * compression/uncompression. Return -1 on failure, 0 on success. */ +int +buf_add_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + const int done) +{ + char *next; + size_t old_avail, avail; + int over = 0; + + do { + int need_new_chunk = 0; + if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) { + size_t cap = data_len / 4; + buf_add_chunk_with_capacity(buf, cap, 1); + } + next = CHUNK_WRITE_PTR(buf->tail); + avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: + over = 1; + break; + case TOR_COMPRESS_ERROR: + return -1; + case TOR_COMPRESS_OK: + if (data_len == 0) { + tor_assert_nonfatal(!done); + over = 1; + } + break; + case TOR_COMPRESS_BUFFER_FULL: + if (avail) { + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ + need_new_chunk = 1; + } + if (data_len == 0 && !done) { + /* We've consumed all the input data, though, so there's no + * point in forging ahead right now. */ + over = 1; + } + break; + } + buf->datalen += old_avail - avail; + buf->tail->datalen += old_avail - avail; + if (need_new_chunk) { + buf_add_chunk_with_capacity(buf, data_len/4, 1); + } + + } while (!over); + check(); + return 0; +} + +/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */ +int +buf_set_to_copy(buf_t **output, + const buf_t *input) +{ + if (*output) + buf_free(*output); + *output = buf_copy(input); + return 0; +} + +/** Log an error and exit if <b>buf</b> is corrupted. + */ +void +buf_assert_ok(buf_t *buf) +{ + tor_assert(buf); + tor_assert(buf->magic == BUFFER_MAGIC); + + if (! buf->head) { + tor_assert(!buf->tail); + tor_assert(buf->datalen == 0); + } else { + chunk_t *ch; + size_t total = 0; + tor_assert(buf->tail); + for (ch = buf->head; ch; ch = ch->next) { + total += ch->datalen; + tor_assert(ch->datalen <= ch->memlen); + tor_assert(ch->data >= &ch->mem[0]); + tor_assert(ch->data <= &ch->mem[0]+ch->memlen); + if (ch->data == &ch->mem[0]+ch->memlen) { + /* LCOV_EXCL_START */ + static int warned = 0; + if (! warned) { + log_warn(LD_BUG, "Invariant violation in buf.c related to #15083"); + warned = 1; + } + /* LCOV_EXCL_STOP */ + } + tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen); + if (!ch->next) + tor_assert(ch == buf->tail); + } + tor_assert(buf->datalen == total); + } +} + diff --git a/src/common/buffers.h b/src/common/buffers.h new file mode 100644 index 0000000000..4275152de2 --- /dev/null +++ b/src/common/buffers.h @@ -0,0 +1,131 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.h + * \brief Header file for buffers.c. + **/ + +#ifndef TOR_BUFFERS_H +#define TOR_BUFFERS_H + +#include "compat.h" +#include "torint.h" +#include "testsupport.h" + +typedef struct buf_t buf_t; + +struct tor_compress_state_t; + +buf_t *buf_new(void); +buf_t *buf_new_with_capacity(size_t size); +size_t buf_get_default_chunk_size(const buf_t *buf); +void buf_free_(buf_t *buf); +#define buf_free(b) FREE_AND_NULL(buf_t, buf_free_, (b)) +void buf_clear(buf_t *buf); +buf_t *buf_copy(const buf_t *buf); + +MOCK_DECL(size_t, buf_datalen, (const buf_t *buf)); +size_t buf_allocation(const buf_t *buf); +size_t buf_slack(const buf_t *buf); + +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + +int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen); + +int buf_add(buf_t *buf, const char *string, size_t string_len); +void buf_add_string(buf_t *buf, const char *string); +void buf_add_printf(buf_t *buf, const char *format, ...) + CHECK_PRINTF(2, 3); +void buf_add_vprintf(buf_t *buf, const char *format, va_list args) + CHECK_PRINTF(2, 0); +int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state, + const char *data, size_t data_len, int done); +int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); +void buf_move_all(buf_t *buf_out, buf_t *buf_in); +void buf_peek(const buf_t *buf, char *string, size_t string_len); +void buf_drain(buf_t *buf, size_t n); +int buf_get_bytes(buf_t *buf, char *string, size_t string_len); +int buf_get_line(buf_t *buf, char *data_out, size_t *data_len); + +#define PEEK_BUF_STARTSWITH_MAX 16 +int buf_peek_startswith(const buf_t *buf, const char *cmd); + +int buf_set_to_copy(buf_t **output, + const buf_t *input); + +void buf_assert_ok(buf_t *buf); + +int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +void buf_pullup(buf_t *buf, size_t bytes, + const char **head_out, size_t *len_out); +char *buf_extract(buf_t *buf, size_t *sz_out); + +#ifdef BUFFERS_PRIVATE +#ifdef TOR_UNIT_TESTS +buf_t *buf_new_with_data(const char *cp, size_t sz); +#endif +size_t buf_preferred_chunk_size(size_t target); + +#define DEBUG_CHUNK_ALLOC +/** A single chunk on a buffer. */ +typedef struct chunk_t { + struct chunk_t *next; /**< The next chunk on the buffer. */ + size_t datalen; /**< The number of bytes stored in this chunk */ + size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ +#ifdef DEBUG_CHUNK_ALLOC + size_t DBG_alloc; +#endif + char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ + uint32_t inserted_time; /**< Timestamp when this chunk was inserted. */ + char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in + * this chunk. */ +} chunk_t; + +/** Magic value for buf_t.magic, to catch pointer errors. */ +#define BUFFER_MAGIC 0xB0FFF312u +/** A resizeable buffer, optimized for reading and writing. */ +struct buf_t { + uint32_t magic; /**< Magic cookie for debugging: Must be set to + * BUFFER_MAGIC. */ + size_t datalen; /**< How many bytes is this buffer holding right now? */ + size_t default_chunk_size; /**< Don't allocate any chunks smaller than + * this for this buffer. */ + chunk_t *head; /**< First chunk in the list, or NULL for none. */ + chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ +}; + +chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped); +/** If a read onto the end of a chunk would be smaller than this number, then + * just start a new chunk. */ +#define MIN_READ_LEN 8 + +/** Return the number of bytes that can be written onto <b>chunk</b> without + * running out of space. */ +static inline size_t +CHUNK_REMAINING_CAPACITY(const chunk_t *chunk) +{ + return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen); +} + +/** Return the next character in <b>chunk</b> onto which data can be appended. + * If the chunk is full, this might be off the end of chunk->mem. */ +static inline char * +CHUNK_WRITE_PTR(chunk_t *chunk) +{ + return chunk->data + chunk->datalen; +} + +#endif /* defined(BUFFERS_PRIVATE) */ + +#endif /* !defined(TOR_BUFFERS_H) */ + diff --git a/src/common/buffers_tls.c b/src/common/buffers_tls.c new file mode 100644 index 0000000000..041f78b818 --- /dev/null +++ b/src/common/buffers_tls.c @@ -0,0 +1,179 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define BUFFERS_PRIVATE +#include "orconfig.h" +#include <stddef.h> +#include "buffers.h" +#include "buffers_tls.h" +#include "compat.h" +#include "compress.h" +#include "util.h" +#include "torint.h" +#include "torlog.h" +#include "tortls.h" +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +/** As read_to_chunk(), but return (negative) error code on error, blocking, + * or TLS, and the number of bytes read otherwise. */ +static inline int +read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, + size_t at_most) +{ + int read_result; + + tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most); + read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most); + if (read_result < 0) + return read_result; + buf->datalen += read_result; + chunk->datalen += read_result; + return read_result; +} + +/** As read_to_buf, but reads from a TLS connection, and returns a TLS + * status value rather than the number of bytes read. + * + * Using TLS on OR connections complicates matters in two ways. + * + * First, a TLS stream has its own read buffer independent of the + * connection's read buffer. (TLS needs to read an entire frame from + * the network before it can decrypt any data. Thus, trying to read 1 + * byte from TLS can require that several KB be read from the network + * and decrypted. The extra data is stored in TLS's decrypt buffer.) + * Because the data hasn't been read by Tor (it's still inside the TLS), + * this means that sometimes a connection "has stuff to read" even when + * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is + * used in connection.c to detect TLS objects with non-empty internal + * buffers and read from them again. + * + * Second, the TLS stream's events do not correspond directly to network + * events: sometimes, before a TLS stream can read, the network must be + * ready to write -- or vice versa. + */ +int +buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most) +{ + int r = 0; + size_t total_read = 0; + + check_no_tls_errors(); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - at_most)) + return -1; + + while (at_most > total_read) { + size_t readlen = at_most - total_read; + chunk_t *chunk; + if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { + chunk = buf_add_chunk_with_capacity(buf, at_most, 1); + if (readlen > chunk->memlen) + readlen = chunk->memlen; + } else { + size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); + chunk = buf->tail; + if (cap < readlen) + readlen = cap; + } + + r = read_to_chunk_tls(buf, chunk, tls, readlen); + if (r < 0) + return r; /* Error */ + tor_assert(total_read+r < INT_MAX); + total_read += r; + if ((size_t)r < readlen) /* eof, block, or no more to read. */ + break; + } + return (int)total_read; +} + +/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk + * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. (Tries to write + * more if there is a forced pending write size.) On success, deduct the + * bytes written from *<b>buf_flushlen</b>. Return the number of bytes + * written on success, and a TOR_TLS error code on failure or blocking. + */ +static inline int +flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, + size_t sz, size_t *buf_flushlen) +{ + int r; + size_t forced; + char *data; + + forced = tor_tls_get_forced_write_size(tls); + if (forced > sz) + sz = forced; + if (chunk) { + data = chunk->data; + tor_assert(sz <= chunk->datalen); + } else { + data = NULL; + tor_assert(sz == 0); + } + r = tor_tls_write(tls, data, sz); + if (r < 0) + return r; + if (*buf_flushlen > (size_t)r) + *buf_flushlen -= r; + else + *buf_flushlen = 0; + buf_drain(buf, r); + log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.", + r,(int)*buf_flushlen,(int)buf->datalen); + return r; +} + +/** As buf_flush_to_socket(), but writes data to a TLS connection. Can write + * more than <b>flushlen</b> bytes. + */ +int +buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen, + size_t *buf_flushlen) +{ + int r; + size_t flushed = 0; + ssize_t sz; + tor_assert(buf_flushlen); + if (BUG(*buf_flushlen > buf->datalen)) { + *buf_flushlen = buf->datalen; + } + if (BUG(flushlen > *buf_flushlen)) { + flushlen = *buf_flushlen; + } + sz = (ssize_t) flushlen; + + /* we want to let tls write even if flushlen is zero, because it might + * have a partial record pending */ + check_no_tls_errors(); + + do { + size_t flushlen0; + if (buf->head) { + if ((ssize_t)buf->head->datalen >= sz) + flushlen0 = sz; + else + flushlen0 = buf->head->datalen; + } else { + flushlen0 = 0; + } + + r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen); + if (r < 0) + return r; + flushed += r; + sz -= r; + if (r == 0) /* Can't flush any more now. */ + break; + } while (sz > 0); + tor_assert(flushed < INT_MAX); + return (int)flushed; +} + diff --git a/src/common/buffers_tls.h b/src/common/buffers_tls.h new file mode 100644 index 0000000000..2f9fda45a0 --- /dev/null +++ b/src/common/buffers_tls.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_BUFFERS_TLS_H +#define TOR_BUFFERS_TLS_H + +struct buf_t; +struct tor_tls_t; + +int buf_read_from_tls(struct buf_t *buf, + struct tor_tls_t *tls, size_t at_most); +int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls, + size_t sz, size_t *buf_flushlen); + +#endif /* !defined(TOR_BUFFERS_TLS_H) */ + diff --git a/src/common/ciphers.inc b/src/common/ciphers.inc index 23f5fd2da4..0084b3e325 100644 --- a/src/common/ciphers.inc +++ b/src/common/ciphers.inc @@ -33,6 +33,26 @@ #else XCIPHER(0xc02f, TLS1_TXT_ECDHE_RSA_WITH_AES_128_GCM_SHA256) #endif +#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + CIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305) +#else + XCIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305) +#endif +#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 + CIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305) +#else + XCIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305) +#endif +#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + CIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) +#else + XCIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) +#endif +#ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + CIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384) +#else + XCIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384) +#endif #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) #else @@ -53,88 +73,28 @@ #else XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA) #endif -#ifdef TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA - CIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA) -#else - XCIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA) -#endif -#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA - CIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA) -#else - XCIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA) -#endif -#ifdef TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA - CIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA) -#else - XCIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA) -#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA) #else XCIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA) #endif -#ifdef TLS1_TXT_DHE_DSS_WITH_AES_128_SHA - CIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA) -#else - XCIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA) -#endif -#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA - CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA) -#else - XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA) -#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) #else XCIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) #endif -#ifdef TLS1_TXT_DHE_DSS_WITH_AES_256_SHA - CIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA) -#else - XCIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA) -#endif -#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA - CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA) -#else - XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA) -#endif -#ifdef SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA - CIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -#else - XCIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -#endif #ifdef TLS1_TXT_RSA_WITH_AES_128_SHA CIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA) #else XCIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA) #endif -#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA - CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA) -#else - XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA) -#endif #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA) #else XCIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA) #endif -#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA - CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA) -#else - XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA) -#endif #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA) #else XCIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA) #endif -#ifdef SSL3_TXT_RSA_RC4_128_SHA - CIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA) -#else - XCIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA) -#endif -#ifdef SSL3_TXT_RSA_RC4_128_MD5 - CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5) -#else - XCIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5) -#endif diff --git a/src/common/compat.c b/src/common/compat.c index 9758751122..f8c4bdb4d8 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -88,21 +88,20 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) while (cnt--) *vcptr++ = 0; } -#endif +#endif /* defined(HAVE_DECL_SECUREZEROMEMORY) && !HAVE_DECL_SECUREZEROMEMORY */ #elif defined(HAVE_READPASSPHRASE_H) #include <readpassphrase.h> #else #include "tor_readpassphrase.h" -#endif +#endif /* defined(_WIN32) || ... */ /* Includes for the process attaching prevention */ #if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) /* Only use the linux prctl; the IRIX prctl is totally different */ #include <sys/prctl.h> #elif defined(__APPLE__) -#include <sys/types.h> #include <sys/ptrace.h> -#endif +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) || ... */ #ifdef HAVE_NETDB_H #include <netdb.h> @@ -116,7 +115,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #ifdef HAVE_SIGNAL_H #include <signal.h> #endif -#ifdef HAVE_SYS_MMAN_H +#ifdef HAVE_MMAP #include <sys/mman.h> #endif #ifdef HAVE_SYS_SYSLIMITS_H @@ -161,7 +160,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode) * are running on one without. */ if (errno != EINVAL) return -1; -#endif +#endif /* defined(O_CLOEXEC) */ log_debug(LD_FS, "Opening %s with flags %x", p, flags); fd = open(p, flags, mode); @@ -173,7 +172,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode) return -1; } } -#endif +#endif /* defined(FD_CLOEXEC) */ return fd; } @@ -191,7 +190,7 @@ tor_fopen_cloexec(const char *path, const char *mode) return NULL; } } -#endif +#endif /* defined(FD_CLOEXEC) */ return result; } @@ -204,16 +203,17 @@ tor_rename(const char *path_old, const char *path_new) sandbox_intern_string(path_new)); } -#if defined(HAVE_SYS_MMAN_H) || defined(RUNNING_DOXYGEN) +#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) /** Try to create a memory mapping for <b>filename</b> and return it. On - * failure, return NULL. Sets errno properly, using ERANGE to mean - * "empty file". */ + * failure, return NULL. Sets errno properly, using ERANGE to mean + * "empty file". Must only be called on trusted Tor-owned files, as changing + * the underlying file's size causes unspecified behavior. */ tor_mmap_t * tor_mmap_file(const char *filename) { int fd; /* router file */ char *string; - int page_size, result; + int result; tor_mmap_t *res; size_t size, filesize; struct stat st; @@ -242,14 +242,13 @@ tor_mmap_file(const char *filename) return NULL; } size = filesize = (size_t)(st.st_size); - /* - * Should we check for weird crap like mmapping a named pipe here, - * or just wait for if (!size) below to fail? - */ - /* ensure page alignment */ - page_size = getpagesize(); - size += (size%page_size) ? page_size-(size%page_size) : 0; + if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) { + log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename); + errno = EFBIG; + close(fd); + return NULL; + } if (!size) { /* Zero-length file. If we call mmap on it, it will succeed but * return NULL, and bad things will happen. So just fail. */ @@ -403,40 +402,8 @@ tor_munmap_file(tor_mmap_t *handle) return 0; } #else -tor_mmap_t * -tor_mmap_file(const char *filename) -{ - struct stat st; - char *res = read_file_to_str(filename, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - tor_mmap_t *handle; - if (! res) - return NULL; - handle = tor_malloc_zero(sizeof(tor_mmap_t)); - handle->data = res; - handle->size = st.st_size; - return handle; -} - -/** Unmap the file mapped with tor_mmap_file(), and return 0 for success - * or -1 for failure. - */ - -int -tor_munmap_file(tor_mmap_t *handle) -{ - char *d = NULL; - if (handle == NULL) - return 0; - - d = (char*)handle->data; - tor_free(d); - memwipe(handle, 0, sizeof(tor_mmap_t)); - tor_free(handle); - - /* Can't fail in this mmap()/munmap()-free case */ - return 0; -} -#endif +#error "cannot implement tor_mmap_file" +#endif /* defined(HAVE_MMAP) || ... || ... */ /** Replacement for snprintf. Differs from platform snprintf in two * ways: First, always NUL-terminates its output. Second, always @@ -576,7 +543,7 @@ tor_vasprintf(char **strp, const char *fmt, va_list args) } *strp = strp_tmp; return len; -#endif +#endif /* defined(HAVE_VASPRINTF) || ... */ } /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at @@ -622,7 +589,7 @@ tor_memmem(const void *_haystack, size_t hlen, } } return NULL; -#endif +#endif /* defined(HAVE_MEMMEM) && (!defined(__GNUC__) || __GNUC__ >= 2) */ } /** @@ -760,7 +727,7 @@ tor_fix_source_file(const char *fname) } return r; } -#endif +#endif /* defined(_WIN32) */ /** * Read a 16-bit value beginning at <b>cp</b>. Equivalent to @@ -854,7 +821,7 @@ replace_file(const char *from, const char *to) return -1; } return tor_rename(from,to); -#endif +#endif /* !defined(_WIN32) */ } /** Change <b>fname</b>'s modification time to now. */ @@ -940,7 +907,7 @@ tor_lockfile_lock(const char *filename, int blocking, int *locked_out) return NULL; } } -#endif +#endif /* defined(_WIN32) || ... */ result = tor_malloc(sizeof(tor_lockfile_t)); result->filename = tor_strdup(filename); @@ -968,7 +935,7 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) } #else /* Closing the lockfile is sufficient. */ -#endif +#endif /* defined(_WIN32) || ... */ close(lockfile->fd); lockfile->fd = -1; @@ -1016,9 +983,9 @@ tor_fd_seekend(int fd) * no need to worry. */ if (rc < 0 && errno == ESPIPE) rc = 0; -#endif +#endif /* defined(ESPIPE) */ return (rc < 0) ? -1 : 0; -#endif +#endif /* defined(_WIN32) */ } /** Move <b>fd</b> to position <b>pos</b> in the file. Return -1 on error, 0 @@ -1057,7 +1024,7 @@ tor_ftruncate(int fd) static bitarray_t *open_sockets = NULL; /** The size of <b>open_sockets</b>, in bits. */ static int max_socket = -1; -#endif +#endif /* defined(DEBUG_SOCKET_COUNTING) */ /** Count of number of sockets currently open. (Undercounts sockets opened by * eventdns and libevent.) */ @@ -1127,7 +1094,7 @@ tor_close_socket,(tor_socket_t s)) tor_assert(open_sockets && s <= max_socket); bitarray_clear(open_sockets, s); } -#endif +#endif /* defined(DEBUG_SOCKET_COUNTING) */ if (r == 0) { --n_sockets_open; } else { @@ -1137,7 +1104,7 @@ tor_close_socket,(tor_socket_t s)) #else if (r != EBADF) --n_sockets_open; // LCOV_EXCL_LINE -- EIO and EINTR too hard to force. -#endif +#endif /* defined(_WIN32) */ r = -1; } @@ -1170,9 +1137,9 @@ mark_socket_open(tor_socket_t s) } bitarray_set(open_sockets, s); } -#else -#define mark_socket_open(s) STMT_NIL -#endif +#else /* !(defined(DEBUG_SOCKET_COUNTING)) */ +#define mark_socket_open(s) ((void) (s)) +#endif /* defined(DEBUG_SOCKET_COUNTING) */ /** @} */ /** As socket(), but counts the number of open sockets. */ @@ -1230,7 +1197,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, * support, we are running on one without. */ if (errno != EINVAL) return s; -#endif /* SOCK_CLOEXEC && SOCK_NONBLOCK */ +#endif /* defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) */ s = socket(domain, type, protocol); if (! SOCKET_OK(s)) @@ -1244,9 +1211,9 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, return TOR_INVALID_SOCKET; } } -#else +#else /* !(defined(FD_CLOEXEC)) */ (void)cloexec; -#endif +#endif /* defined(FD_CLOEXEC) */ if (nonblock) { if (set_socket_nonblocking(s) == -1) { @@ -1258,11 +1225,22 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, goto socket_ok; /* So that socket_ok will not be unused. */ socket_ok: + tor_take_socket_ownership(s); + return s; +} + +/** + * For socket accounting: remember that we are the owner of the socket + * <b>s</b>. This will prevent us from overallocating sockets, and prevent us + * from asserting later when we close the socket <b>s</b>. + */ +void +tor_take_socket_ownership(tor_socket_t s) +{ socket_accounting_lock(); ++n_sockets_open; mark_socket_open(s); socket_accounting_unlock(); - return s; } /** As accept(), but counts the number of open sockets. */ @@ -1302,7 +1280,8 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } -#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) +#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) \ + && defined(SOCK_NONBLOCK) int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) | (nonblock ? SOCK_NONBLOCK : 0); s = accept4(sockfd, addr, len, ext_flags); @@ -1314,7 +1293,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, * we are missing SOCK_CLOEXEC/SOCK_NONBLOCK support. */ if (errno != EINVAL && errno != ENOSYS) return s; -#endif +#endif /* defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) ... */ s = accept(sockfd, addr, len); if (!SOCKET_OK(s)) @@ -1328,9 +1307,9 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } } -#else +#else /* !(defined(FD_CLOEXEC)) */ (void)cloexec; -#endif +#endif /* defined(FD_CLOEXEC) */ if (nonblock) { if (set_socket_nonblocking(s) == -1) { @@ -1342,10 +1321,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, goto socket_ok; /* So that socket_ok will not be unused. */ socket_ok: - socket_accounting_lock(); - ++n_sockets_open; - mark_socket_open(s); - socket_accounting_unlock(); + tor_take_socket_ownership(s); return s; } @@ -1368,6 +1344,24 @@ tor_getsockname,(tor_socket_t sock, struct sockaddr *address, return getsockname(sock, address, address_len); } +/** + * Find the local address associated with the socket <b>sock</b>, and + * place it in *<b>addr_out</b>. Return 0 on success, -1 on failure. + * + * (As tor_getsockname, but instead places the result in a tor_addr_t.) */ +int +tor_addr_from_getsockname(tor_addr_t *addr_out, tor_socket_t sock) +{ + struct sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + memset(&ss, 0, sizeof(ss)); + + if (tor_getsockname(sock, (struct sockaddr *) &ss, &ss_len) < 0) + return -1; + + return tor_addr_from_sockaddr(addr_out, (struct sockaddr *)&ss, NULL); +} + /** Turn <b>socket</b> into a nonblocking socket. Return 0 on success, -1 * on failure. */ @@ -1390,7 +1384,7 @@ set_socket_nonblocking(tor_socket_t sock) log_warn(LD_NET, "Couldn't set file status flags: %s", strerror(errno)); return -1; } -#endif +#endif /* defined(_WIN32) */ return 0; } @@ -1428,7 +1422,7 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) * are running on one without. */ if (errno != EINVAL) return -errno; -#endif +#endif /* defined(SOCK_CLOEXEC) */ r = socketpair(family, type, protocol, fd); if (r < 0) @@ -1451,7 +1445,7 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) return -errno; } } -#endif +#endif /* defined(FD_CLOEXEC) */ goto sockets_ok; /* So that sockets_ok will not be unused. */ sockets_ok: @@ -1467,9 +1461,9 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) socket_accounting_unlock(); return 0; -#else +#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */ return tor_ersatz_socketpair(family, type, protocol, fd); -#endif +#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */ } #ifdef NEED_ERSATZ_SOCKETPAIR @@ -1626,7 +1620,7 @@ tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) #undef SIZEOF_SOCKADDR -#endif +#endif /* defined(NEED_ERSATZ_SOCKETPAIR) */ /* Return the maximum number of allowed sockets. */ int @@ -1680,7 +1674,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) #else const char *platform = "unknown platforms with no getrlimit()"; const unsigned long MAX_CONNECTIONS = 15000; -#endif +#endif /* defined(CYGWIN) || defined(__CYGWIN__) || ... */ log_fn(LOG_INFO, LD_NET, "This platform is missing getrlimit(). Proceeding."); if (limit > MAX_CONNECTIONS) { @@ -1691,7 +1685,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) return -1; } limit = MAX_CONNECTIONS; -#else /* HAVE_GETRLIMIT */ +#else /* !(!defined(HAVE_GETRLIMIT)) */ struct rlimit rlim; if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { @@ -1741,7 +1735,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) couldnt_set = 0; } } -#endif /* OPEN_MAX */ +#endif /* defined(OPEN_MAX) */ if (couldnt_set) { log_warn(LD_CONFIG,"Couldn't set maximum number of file descriptors: %s", strerror(setrlimit_errno)); @@ -1749,7 +1743,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) } /* leave some overhead for logs, etc, */ limit = rlim.rlim_cur; -#endif /* HAVE_GETRLIMIT */ +#endif /* !defined(HAVE_GETRLIMIT) */ if (limit > INT_MAX) limit = INT_MAX; @@ -1787,7 +1781,7 @@ log_credential_status(void) "UID is %u (real), %u (effective), %u (saved)", (unsigned)ruid, (unsigned)euid, (unsigned)suid); } -#else +#else /* !(defined(HAVE_GETRESUID)) */ /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */ ruid = getuid(); euid = geteuid(); @@ -1796,7 +1790,7 @@ log_credential_status(void) log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "UID is %u (real), %u (effective), unknown (saved)", (unsigned)ruid, (unsigned)euid); -#endif +#endif /* defined(HAVE_GETRESUID) */ /* log GIDs */ #ifdef HAVE_GETRESGID @@ -1808,7 +1802,7 @@ log_credential_status(void) "GID is %u (real), %u (effective), %u (saved)", (unsigned)rgid, (unsigned)egid, (unsigned)sgid); } -#else +#else /* !(defined(HAVE_GETRESGID)) */ /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */ rgid = getgid(); egid = getegid(); @@ -1816,7 +1810,7 @@ log_credential_status(void) log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "GID is %u (real), %u (effective), unknown (saved)", (unsigned)rgid, (unsigned)egid); -#endif +#endif /* defined(HAVE_GETRESGID) */ /* log supplementary groups */ sup_gids_size = 64; @@ -1856,7 +1850,7 @@ log_credential_status(void) return 0; } -#endif +#endif /* !defined(_WIN32) */ #ifndef _WIN32 /** Cached struct from the last getpwname() call we did successfully. */ @@ -1881,9 +1875,12 @@ tor_passwd_dup(const struct passwd *pw) return new_pw; } +#define tor_passwd_free(pw) \ + FREE_AND_NULL(struct passwd, tor_passwd_free_, (pw)) + /** Helper: free one of our cached 'struct passwd' values. */ static void -tor_passwd_free(struct passwd *pw) +tor_passwd_free_(struct passwd *pw) { if (!pw) return; @@ -1956,7 +1953,7 @@ tor_getpwuid(uid_t uid) return NULL; } -#endif +#endif /* !defined(_WIN32) */ /** Return true iff we were compiled with capability support, and capabilities * seem to work. **/ @@ -1969,9 +1966,9 @@ have_capability_support(void) return 0; cap_free(caps); return 1; -#else +#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ return 0; -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ } #ifdef HAVE_LINUX_CAPABILITIES @@ -2030,7 +2027,7 @@ drop_capabilities(int pre_setuid) return 0; } -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ /** Call setuid and setgid to run as <b>user</b> and switch to their * primary group. Return 0 on success. On failure, log and return -1. @@ -2080,13 +2077,13 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(1)) return -1; } -#else +#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ (void) keep_bindlow; if (warn_if_no_caps) { log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support " "on this system."); } -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ /* Properly switch egid,gid,euid,uid here or bail out */ if (setgroups(1, &pw->pw_gid)) { @@ -2146,7 +2143,7 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(0)) return -1; } -#endif +#endif /* defined(HAVE_LINUX_CAPABILITIES) */ #if !defined(CYGWIN) && !defined(__CYGWIN__) /* If we tried to drop privilege to a group/user other than root, attempt to @@ -2170,7 +2167,7 @@ switch_id(const char *user, const unsigned flags) return -1; } } -#endif +#endif /* !defined(CYGWIN) && !defined(__CYGWIN__) */ /* Check what really happened */ if (log_credential_status()) { @@ -2179,8 +2176,8 @@ switch_id(const char *user, const unsigned flags) have_already_switched_id = 1; /* mark success so we never try again */ -#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL) -#ifdef PR_SET_DUMPABLE +#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && \ + defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE) if (pw->pw_uid) { /* Re-enable core dumps if we're not running as root. */ log_info(LD_CONFIG, "Re-enabling coredumps"); @@ -2188,17 +2185,16 @@ switch_id(const char *user, const unsigned flags) log_warn(LD_CONFIG, "Unable to re-enable coredumps: %s",strerror(errno)); } } -#endif -#endif +#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */ return 0; -#else +#else /* !(!defined(_WIN32)) */ (void)user; (void)flags; log_warn(LD_CONFIG, "Switching users is unsupported on your OS."); return -1; -#endif +#endif /* !defined(_WIN32) */ } /* We only use the linux prctl for now. There is no Win32 support; this may @@ -2221,35 +2217,32 @@ switch_id(const char *user, const unsigned flags) int tor_disable_debugger_attach(void) { - int r, attempted; - r = -1; - attempted = 0; + int r = -1; log_debug(LD_CONFIG, "Attemping to disable debugger attachment to Tor for " "unprivileged users."); -#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL) -#ifdef PR_SET_DUMPABLE - attempted = 1; +#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) \ + && defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE) +#define TRIED_TO_DISABLE r = prctl(PR_SET_DUMPABLE, 0); -#endif -#endif -#if defined(__APPLE__) && defined(PT_DENY_ATTACH) - if (r < 0) { - attempted = 1; - r = ptrace(PT_DENY_ATTACH, 0, 0, 0); - } -#endif +#elif defined(__APPLE__) && defined(PT_DENY_ATTACH) +#define TRIED_TO_ATTACH + r = ptrace(PT_DENY_ATTACH, 0, 0, 0); +#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) ... || ... */ // XXX: TODO - Mac OS X has dtrace and this may be disabled. // XXX: TODO - Windows probably has something similar - if (r == 0 && attempted) { +#ifdef TRIED_TO_DISABLE + if (r == 0) { log_debug(LD_CONFIG,"Debugger attachment disabled for " "unprivileged users."); return 1; - } else if (attempted) { + } else { log_warn(LD_CONFIG, "Unable to disable debugger attaching: %s", strerror(errno)); } +#endif /* defined(TRIED_TO_DISABLE) */ +#undef TRIED_TO_DISABLE return r; } @@ -2268,7 +2261,7 @@ get_user_homedir(const char *username) } return tor_strdup(pw->pw_dir); } -#endif +#endif /* defined(HAVE_PWD_H) */ /** Modify <b>fname</b> to contain the name of its parent directory. Doesn't * actually examine the filesystem; does a purely syntactic modification. @@ -2296,7 +2289,7 @@ get_parent_directory(char *fname) if (fname[0] && fname[1] == ':') { fname += 2; } -#endif +#endif /* defined(_WIN32) */ /* Now we want to remove all path-separators at the end of the string, * and to remove the end of the string starting with the path separator * before the last non-path-separator. In perl, this would be @@ -2335,17 +2328,36 @@ get_parent_directory(char *fname) static char * alloc_getcwd(void) { -#ifdef PATH_MAX -#define MAX_CWD PATH_MAX -#else -#define MAX_CWD 4096 -#endif +#ifdef HAVE_GET_CURRENT_DIR_NAME + /* Glibc makes this nice and simple for us. */ + char *cwd = get_current_dir_name(); + char *result = NULL; + if (cwd) { + /* We make a copy here, in case tor_malloc() is not malloc(). */ + result = tor_strdup(cwd); + raw_free(cwd); // alias for free to avoid tripping check-spaces. + } + return result; +#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */ + size_t size = 1024; + char *buf = NULL; + char *ptr = NULL; + + while (ptr == NULL) { + buf = tor_realloc(buf, size); + ptr = getcwd(buf, size); + + if (ptr == NULL && errno != ERANGE) { + tor_free(buf); + return NULL; + } - char path_buf[MAX_CWD]; - char *path = getcwd(path_buf, sizeof(path_buf)); - return path ? tor_strdup(path) : NULL; + size *= 2; + } + return buf; +#endif /* defined(HAVE_GET_CURRENT_DIR_NAME) */ } -#endif +#endif /* !defined(_WIN32) */ /** Expand possibly relative path <b>fname</b> to an absolute path. * Return a newly allocated string, possibly equal to <b>fname</b>. */ @@ -2361,7 +2373,7 @@ make_path_absolute(char *fname) if (absfname_malloced) raw_free(absfname_malloced); return absfname; -#else +#else /* !(defined(_WIN32)) */ char *absfname = NULL, *path = NULL; tor_assert(fname); @@ -2384,7 +2396,7 @@ make_path_absolute(char *fname) } } return absfname; -#endif +#endif /* defined(_WIN32) */ } #ifndef HAVE__NSGETENVIRON @@ -2393,8 +2405,8 @@ make_path_absolute(char *fname) #ifndef RUNNING_DOXYGEN extern char **environ; #endif -#endif -#endif +#endif /* !defined(HAVE_EXTERN_ENVIRON_DECLARED) */ +#endif /* !defined(HAVE__NSGETENVIRON) */ /** Return the current environment. This is a portable replacement for * 'environ'. */ @@ -2406,14 +2418,14 @@ get_environment(void) * when we do a mostly-static build on OSX 10.7, the resulting binary won't * work on OSX 10.6. */ return *_NSGetEnviron(); -#else +#else /* !(defined(HAVE__NSGETENVIRON)) */ return environ; -#endif +#endif /* defined(HAVE__NSGETENVIRON) */ } /** Get name of current host and write it to <b>name</b> array, whose * length is specified by <b>namelen</b> argument. Return 0 upon - * successfull completion; otherwise return return -1. (Currently, + * successful completion; otherwise return return -1. (Currently, * this function is merely a mockable wrapper for POSIX gethostname().) */ MOCK_IMPL(int, @@ -2549,6 +2561,7 @@ tor_inet_pton(int af, const char *src, void *dst) int gapPos = -1, i, setWords=0; const char *dot = strchr(src, '.'); const char *eow; /* end of words. */ + memset(words, 0xf8, sizeof(words)); if (dot == src) return 0; else if (!dot) @@ -2586,7 +2599,7 @@ tor_inet_pton(int af, const char *src, void *dst) long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd - * where treats "0xfoo" as an error, rather than as "0" followed by + * which treat "0xfoo" as an error, rather than as "0" followed by * "xfoo". */ return 0; } @@ -2673,7 +2686,8 @@ static int uname_result_is_set = 0; /** Return a pointer to a description of our platform. */ -MOCK_IMPL(const char *, get_uname, (void)) +MOCK_IMPL(const char *, +get_uname,(void)) { #ifdef HAVE_UNAME struct utsname u; @@ -2684,7 +2698,7 @@ MOCK_IMPL(const char *, get_uname, (void)) /* (Linux says 0 is success, Solaris says 1 is success) */ strlcpy(uname_result, u.sysname, sizeof(uname_result)); } else -#endif +#endif /* defined(HAVE_UNAME) */ { #ifdef _WIN32 OSVERSIONINFOEX info; @@ -2780,11 +2794,11 @@ MOCK_IMPL(const char *, get_uname, (void)) if (!is_server && !is_client) { strlcat(uname_result, " [client or server]", sizeof(uname_result)); } -#else +#else /* !(defined(_WIN32)) */ /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); /* LCOV_EXCL_STOP */ -#endif +#endif /* defined(_WIN32) */ } uname_result_is_set = 1; } @@ -2840,7 +2854,7 @@ compute_num_cpus_impl(void) return -1; #else return -1; -#endif +#endif /* defined(_WIN32) || ... */ } #define MAX_DETECTABLE_CPUS 16 @@ -2879,7 +2893,7 @@ compute_num_cpus(void) /** Helper: Deal with confused or out-of-bounds values from localtime_r and * friends. (On some platforms, they can give out-of-bounds values or can * return NULL.) If <b>islocal</b>, this is a localtime result; otherwise - * it's from gmtime. The function returned <b>r</b>, when given <b>timep</b> + * it's from gmtime. The function returns <b>r</b>, when given <b>timep</b> * as its input. If we need to store new results, store them in * <b>resultbuf</b>. */ static struct tm * @@ -3003,7 +3017,7 @@ tor_localtime_r(const time_t *timep, struct tm *result) memcpy(result, r, sizeof(struct tm)); return correct_tm(1, timep, result, r); } -#endif +#endif /* defined(HAVE_LOCALTIME_R) || ... */ /** @} */ /** @{ */ @@ -3046,9 +3060,13 @@ tor_gmtime_r(const time_t *timep, struct tm *result) memcpy(result, r, sizeof(struct tm)); return correct_tm(0, timep, result, r); } -#endif +#endif /* defined(HAVE_GMTIME_R) || ... */ #if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +#define HAVE_UNIX_MLOCKALL +#endif + +#ifdef HAVE_UNIX_MLOCKALL /** Attempt to raise the current and max rlimit to infinity for our process. * This only needs to be done once and can probably only be done when we have * not already dropped privileges. @@ -3079,7 +3097,7 @@ tor_set_max_memlock(void) return 0; } -#endif +#endif /* defined(HAVE_UNIX_MLOCKALL) */ /** Attempt to lock all current and all future memory pages. * This should only be called once and while we're privileged. @@ -3104,7 +3122,7 @@ tor_mlockall(void) * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx */ -#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +#ifdef HAVE_UNIX_MLOCKALL if (tor_set_max_memlock() == 0) { log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY."); } @@ -3125,10 +3143,10 @@ tor_mlockall(void) "pages: %s", strerror(errno)); return -1; } -#else +#else /* !(defined(HAVE_UNIX_MLOCKALL)) */ log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); return -1; -#endif +#endif /* defined(HAVE_UNIX_MLOCKALL) */ } /** @@ -3156,7 +3174,7 @@ tor_socket_errno(tor_socket_t sock) } return err; } -#endif +#endif /* defined(_WIN32) */ #if defined(_WIN32) #define E(code, s) { code, (s " [" #code " ]") } @@ -3232,7 +3250,7 @@ tor_socket_strerror(int e) } return strerror(e); } -#endif +#endif /* defined(_WIN32) */ /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -3258,7 +3276,7 @@ network_init(void) /* WSAData.iMaxSockets might show the max sockets we're allowed to use. * We might use it to complain if we're trying to be a server but have * too few sockets available. */ -#endif +#endif /* defined(_WIN32) */ return 0; } @@ -3279,7 +3297,7 @@ format_win32_error(DWORD err) FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), (LPVOID)&str, 0, NULL); @@ -3294,9 +3312,9 @@ format_win32_error(DWORD err) result = tor_malloc(len); wcstombs(result,str,len); result[len-1] = '\0'; -#else +#else /* !(defined(UNICODE)) */ result = tor_strdup(str); -#endif +#endif /* defined(UNICODE) */ } else { result = tor_strdup("<unformattable error>"); } @@ -3305,7 +3323,7 @@ format_win32_error(DWORD err) } return result; } -#endif +#endif /* defined(_WIN32) */ #if defined(HW_PHYSMEM64) /* This appears to be an OpenBSD thing */ @@ -3313,7 +3331,7 @@ format_win32_error(DWORD err) #elif defined(HW_MEMSIZE) /* OSX defines this one */ #define INT64_HW_MEM HW_MEMSIZE -#endif +#endif /* defined(HW_PHYSMEM64) || ... */ /** * Helper: try to detect the total system memory, and return it. On failure, @@ -3346,8 +3364,8 @@ get_total_system_memory_impl(void) tor_free(s); return result * 1024; - err: /* LCOV_EXCL_START Can't reach this unless proc is broken. */ + err: tor_free(s); close(fd); return 0; @@ -3387,15 +3405,15 @@ get_total_system_memory_impl(void) #else /* I have no clue. */ return 0; -#endif +#endif /* defined(__linux__) || ... */ } /** * Try to find out how much physical memory the system has. On success, * return 0 and set *<b>mem_out</b> to that value. On failure, return -1. */ -int -get_total_system_memory(size_t *mem_out) +MOCK_IMPL(int, +get_total_system_memory, (size_t *mem_out)) { static size_t mem_cached=0; uint64_t m = get_total_system_memory_impl(); @@ -3420,7 +3438,7 @@ get_total_system_memory(size_t *mem_out) * size_t. */ m = SIZE_MAX; } -#endif +#endif /* SIZE_MAX != UINT64_MAX */ *mem_out = mem_cached = (size_t) m; @@ -3501,7 +3519,7 @@ tor_getpass(const char *prompt, char *output, size_t buflen) return r; #else #error "No implementation for tor_getpass found!" -#endif +#endif /* defined(HAVE_READPASSPHRASE) || ... */ } /** Return the amount of free disk space we have permission to use, in @@ -3541,6 +3559,6 @@ tor_get_avail_disk_space(const char *path) (void)path; errno = ENOSYS; return -1; -#endif +#endif /* defined(HAVE_STATVFS) || ... */ } diff --git a/src/common/compat.h b/src/common/compat.h index ee1c9454de..c7e7f8d9ef 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_H @@ -10,6 +10,9 @@ #ifdef _WIN32 #include <winsock2.h> #include <ws2tcpip.h> +#ifndef SIO_IDEAL_SEND_BACKLOG_QUERY +#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b +#endif #endif #include "torint.h" #include "testsupport.h" @@ -50,8 +53,8 @@ * clang rejects because it is off the end of a less-than-3. Clang hates this, * even though those references never actually happen. */ # undef strcmp -# endif -#endif +#endif /* __has_feature(address_sanitizer) */ +#endif /* defined(__has_feature) */ #include <stdio.h> #include <errno.h> @@ -76,13 +79,13 @@ __attribute__ ((format(printf, formatIdx, firstArg))) #else #define CHECK_PRINTF(formatIdx, firstArg) -#endif +#endif /* defined(__GNUC__) */ #ifdef __GNUC__ #define CHECK_SCANF(formatIdx, firstArg) \ __attribute__ ((format(scanf, formatIdx, firstArg))) #else #define CHECK_SCANF(formatIdx, firstArg) -#endif +#endif /* defined(__GNUC__) */ /* What GCC do we have? */ #ifdef __GNUC__ @@ -109,18 +112,18 @@ PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(pop) -# else +#else /* !(defined(__clang__) || GCC_VERSION >= 406) */ /* older version of gcc: no push/pop support. */ # define DISABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) -# endif -#else /* ifdef __GNUC__ */ +#endif /* defined(__clang__) || GCC_VERSION >= 406 */ +#else /* !(defined(__GNUC__)) */ /* not gcc at all */ # define DISABLE_GCC_WARNING(warning) # define ENABLE_GCC_WARNING(warning) -#endif +#endif /* defined(__GNUC__) */ /* inline is __inline on windows. */ #ifdef _WIN32 @@ -142,9 +145,9 @@ #define __func__ __FUNC__ #else #define __func__ "???" -#endif -#endif /* ifndef MAVE_MACRO__func__ */ -#endif /* if not windows */ +#endif /* defined(HAVE_MACRO__FUNCTION__) || ... */ +#endif /* !defined(HAVE_MACRO__func__) */ +#endif /* defined(_MSC_VER) */ #define U64_TO_DBL(x) ((double) (x)) #define DBL_TO_U64(x) ((uint64_t) (x)) @@ -157,7 +160,7 @@ * problems), but if enumerated types are unsigned, we must use unsigned, * so that the loss of precision doesn't make large values negative. */ #define ENUM_BF(t) t -#endif +#endif /* defined(ENUM_VALS_ARE_SIGNED) */ /* GCC has several useful attributes. */ #if defined(__GNUC__) && __GNUC__ >= 3 @@ -194,7 +197,7 @@ * taken. This can generate slightly better code with some CPUs. */ #define PREDICT_UNLIKELY(exp) __builtin_expect(!!(exp), 0) -#else +#else /* !(defined(__GNUC__) && __GNUC__ >= 3) */ #define ATTR_NORETURN #define ATTR_CONST #define ATTR_MALLOC @@ -204,7 +207,7 @@ #define ATTR_WUR #define PREDICT_LIKELY(exp) (exp) #define PREDICT_UNLIKELY(exp) (exp) -#endif +#endif /* defined(__GNUC__) && __GNUC__ >= 3 */ /** Expands to a syntactically valid empty statement. */ #define STMT_NIL (void)0 @@ -224,7 +227,7 @@ #else #define STMT_BEGIN do { #define STMT_END } while (0) -#endif +#endif /* defined(__GNUC__) || ... */ /* Some tools (like coccinelle) don't like to see operators as macro * arguments. */ @@ -251,7 +254,7 @@ */ #undef strlcat #undef strlcpy -#endif +#endif /* defined __APPLE__ */ #ifndef HAVE_STRLCAT size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); @@ -272,24 +275,28 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); #define I64_PRINTF_ARG(a) (a) #define I64_SCANF_ARG(a) (a) #define I64_LITERAL(n) (n ## i64) -#else +#else /* !(defined(_MSC_VER)) */ #define U64_PRINTF_ARG(a) ((long long unsigned int)(a)) #define U64_SCANF_ARG(a) ((long long unsigned int*)(a)) #define U64_LITERAL(n) (n ## llu) #define I64_PRINTF_ARG(a) ((long long signed int)(a)) #define I64_SCANF_ARG(a) ((long long signed int*)(a)) #define I64_LITERAL(n) (n ## ll) +#endif /* defined(_MSC_VER) */ + +#if defined(__MINGW32__) || defined(__MINGW64__) +#define MINGW_ANY #endif -#if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) +#if defined(_MSC_VER) || defined(MINGW_ANY) /** The formatting string used to put a uint64_t value in a printf() or * scanf() function. See also U64_PRINTF_ARG and U64_SCANF_ARG. */ #define U64_FORMAT "%I64u" #define I64_FORMAT "%I64d" -#else +#else /* !(defined(_MSC_VER) || defined(MINGW_ANY)) */ #define U64_FORMAT "%llu" #define I64_FORMAT "%lld" -#endif +#endif /* defined(_MSC_VER) || defined(MINGW_ANY) */ #if (SIZEOF_INTPTR_T == SIZEOF_INT) #define INTPTR_T_FORMAT "%d" @@ -302,7 +309,7 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); #define INTPTR_PRINTF_ARG(x) I64_PRINTF_ARG(x) #else #error Unknown: SIZEOF_INTPTR_T -#endif +#endif /* (SIZEOF_INTPTR_T == SIZEOF_INT) || ... */ /** Represents an mmaped file. Allocated via tor_mmap_file; freed with * tor_munmap_file. */ @@ -311,12 +318,12 @@ typedef struct tor_mmap_t { size_t size; /**< Size of the file. */ /* None of the fields below should be accessed from outside compat.c */ -#ifdef HAVE_SYS_MMAN_H +#ifdef HAVE_MMAP size_t mapping_size; /**< Size of the actual mapping. (This is this file * size, rounded up to the nearest page.) */ #elif defined _WIN32 HANDLE mmap_handle; -#endif +#endif /* defined(HAVE_MMAP) || ... */ } tor_mmap_t; @@ -378,7 +385,7 @@ const char *tor_fix_source_file(const char *fname); #else #define SHORT_FILE__ (__FILE__) #define tor_fix_source_file(s) (s) -#endif +#endif /* defined(_WIN32) */ /* ===== Time compatibility */ @@ -397,7 +404,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); (tvout)->tv_sec++; \ } \ } while (0) -#endif +#endif /* !defined(timeradd) */ #ifndef timersub /** Replacement for timersub on platforms that do not have it: sets tvout to @@ -411,13 +418,13 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); (tvout)->tv_sec--; \ } \ } while (0) -#endif +#endif /* !defined(timersub) */ #ifndef timercmp -/** Replacement for timersub on platforms that do not have it: returns true +/** Replacement for timercmp on platforms that do not have it: returns true * iff the relational operator "op" makes the expression tv1 op tv2 true. * - * Note that while this definition should work for all boolean opeators, some + * Note that while this definition should work for all boolean operators, some * platforms' native timercmp definitions do not support >=, <=, or ==. So * don't use those. */ @@ -425,7 +432,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); (((tv1)->tv_sec == (tv2)->tv_sec) ? \ ((tv1)->tv_usec op (tv2)->tv_usec) : \ ((tv1)->tv_sec op (tv2)->tv_sec)) -#endif +#endif /* !defined(timercmp) */ /* ===== File compatibility */ int tor_open_cloexec(const char *path, int flags, unsigned mode); @@ -467,7 +474,7 @@ typedef int socklen_t; #define TOR_SOCKET_T_FORMAT INTPTR_T_FORMAT #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET) #define TOR_INVALID_SOCKET INVALID_SOCKET -#else +#else /* !(defined(_WIN32)) */ /** Type used for a network socket. */ #define tor_socket_t int #define TOR_SOCKET_T_FORMAT "%d" @@ -475,10 +482,11 @@ typedef int socklen_t; #define SOCKET_OK(s) ((s) >= 0) /** Error/uninitialized value for a tor_socket_t. */ #define TOR_INVALID_SOCKET (-1) -#endif +#endif /* defined(_WIN32) */ int tor_close_socket_simple(tor_socket_t s); MOCK_DECL(int, tor_close_socket, (tor_socket_t s)); +void tor_take_socket_ownership(tor_socket_t s); tor_socket_t tor_open_socket_with_extensions( int domain, int type, int protocol, int cloexec, int nonblock); @@ -502,6 +510,8 @@ int get_n_open_sockets(void); MOCK_DECL(int, tor_getsockname,(tor_socket_t socket, struct sockaddr *address, socklen_t *address_len)); +struct tor_addr_t; +int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags) #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags) @@ -522,19 +532,19 @@ struct in6_addr #define s6_addr16 in6_u.u6_addr16 #define s6_addr32 in6_u.u6_addr32 }; -#endif +#endif /* !defined(HAVE_STRUCT_IN6_ADDR) */ /** @{ */ /** Many BSD variants seem not to define these. */ -#if defined(__APPLE__) || defined(__darwin__) || defined(__FreeBSD__) \ - || defined(__NetBSD__) || defined(__OpenBSD__) +#if defined(__APPLE__) || defined(__darwin__) || \ + defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) #ifndef s6_addr16 #define s6_addr16 __u6_addr.__u6_addr16 #endif #ifndef s6_addr32 #define s6_addr32 __u6_addr.__u6_addr32 #endif -#endif +#endif /* defined(__APPLE__) || defined(__darwin__) || ... */ /** @} */ #ifndef HAVE_SA_FAMILY_T @@ -566,7 +576,7 @@ struct sockaddr_in6 { struct in6_addr sin6_addr; // uint32_t sin6_scope_id; }; -#endif +#endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */ MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen)); int tor_inet_aton(const char *cp, struct in_addr *addr) ATTR_NONNULL((1,2)); @@ -607,14 +617,14 @@ int network_init(void); #define ERRNO_IS_EINTR(e) ((e) == WSAEINTR || 0) int tor_socket_errno(tor_socket_t sock); const char *tor_socket_strerror(int e); -#else +#else /* !(defined(_WIN32)) */ #define SOCK_ERRNO(e) e #if EAGAIN == EWOULDBLOCK /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ #define ERRNO_IS_EAGAIN(e) ((e) == EAGAIN || 0) #else #define ERRNO_IS_EAGAIN(e) ((e) == EAGAIN || (e) == EWOULDBLOCK) -#endif +#endif /* EAGAIN == EWOULDBLOCK */ #define ERRNO_IS_EINTR(e) ((e) == EINTR || 0) #define ERRNO_IS_EINPROGRESS(e) ((e) == EINPROGRESS || 0) #define ERRNO_IS_CONN_EINPROGRESS(e) ((e) == EINPROGRESS || 0) @@ -625,7 +635,7 @@ const char *tor_socket_strerror(int e); #define ERRNO_IS_EADDRINUSE(e) (((e) == EADDRINUSE) || 0) #define tor_socket_errno(sock) (errno) #define tor_socket_strerror(e) strerror(e) -#endif +#endif /* defined(_WIN32) */ /** Specified SOCKS5 status codes. */ typedef enum { @@ -691,7 +701,7 @@ char *make_path_absolute(char *fname); char **get_environment(void); -int get_total_system_memory(size_t *mem_out); +MOCK_DECL(int, get_total_system_memory, (size_t *mem_out)); int compute_num_cpus(void); @@ -728,7 +738,7 @@ char *format_win32_error(DWORD err); #define VER_SUITE_SINGLEUSERTS 0x00000100 #endif -#endif +#endif /* defined(_WIN32) */ #ifdef COMPAT_PRIVATE #if !defined(HAVE_SOCKETPAIR) || defined(_WIN32) || defined(TOR_UNIT_TESTS) @@ -736,12 +746,12 @@ char *format_win32_error(DWORD err); STATIC int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); #endif -#endif +#endif /* defined(COMPAT_PRIVATE) */ ssize_t tor_getpass(const char *prompt, char *output, size_t buflen); /* This needs some of the declarations above so we include it here. */ #include "compat_threads.h" -#endif +#endif /* !defined(TOR_COMPAT_H) */ diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 4a3b1af922..e60eb148d8 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,7 +11,7 @@ #define COMPAT_LIBEVENT_PRIVATE #include "compat_libevent.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "torlog.h" @@ -69,7 +69,7 @@ suppress_libevent_log_msg(const char *msg) /* Wrapper for event_free() that tolerates tor_event_free(NULL) */ void -tor_event_free(struct event *ev) +tor_event_free_(struct event *ev) { if (ev == NULL) return; @@ -79,6 +79,43 @@ tor_event_free(struct event *ev) /** Global event base for use by the main thread. */ static struct event_base *the_event_base = NULL; +/** + * @defgroup postloop post-loop event helpers + * + * If we're not careful, Libevent can susceptible to infinite event chains: + * one event can activate another, whose callback activates another, whose + * callback activates another, ad infinitum. While this is happening, + * Libevent won't be checking timeouts, socket-based events, signals, and so + * on. + * + * We solve this problem by marking some events as "post-loop". A post-loop + * event behaves like any ordinary event, but any events that _it_ activates + * cannot run until Libevent has checked for other events at least once. + * + * @{ */ + +/** + * An event that stops Libevent from running any more events on the current + * iteration of its loop, until it has re-checked for socket events, signal + * events, timeouts, etc. + */ +static struct event *rescan_mainloop_ev = NULL; + +/** + * Callback to implement rescan_mainloop_ev: it simply exits the mainloop, + * and relies on Tor to re-enter the mainloop since no error has occurred. + */ +static void +rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg) +{ + (void)fd; + (void)events; + struct event_base *the_base = arg; + event_base_loopbreak(the_base); +} + +/** @} */ + /* This is what passes for version detection on OSX. We set * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before * 10.4.0 (aka 1040). */ @@ -88,8 +125,8 @@ static struct event_base *the_event_base = NULL; (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) #else #define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif +#endif /* defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) */ +#endif /* defined(__APPLE__) */ /** Initialize the Libevent library and set up the event base. */ void @@ -126,7 +163,16 @@ tor_libevent_initialize(tor_libevent_cfg *torcfg) if (!the_event_base) { /* LCOV_EXCL_START */ log_err(LD_GENERAL, "Unable to initialize Libevent: cannot continue."); - exit(1); + exit(1); // exit ok: libevent is broken. + /* LCOV_EXCL_STOP */ + } + + rescan_mainloop_ev = event_new(the_event_base, -1, 0, + rescan_mainloop_cb, the_event_base); + if (!rescan_mainloop_ev) { + /* LCOV_EXCL_START */ + log_err(LD_GENERAL, "Unable to create rescan event: cannot continue."); + exit(1); // exit ok: libevent is broken. /* LCOV_EXCL_STOP */ } @@ -207,13 +253,42 @@ periodic_timer_new(struct event_base *base, } timer->cb = cb; timer->data = data; - event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/ + periodic_timer_launch(timer, tv); return timer; } +/** + * Launch the timer <b>timer</b> to run at <b>tv</b> from now, and every + * <b>tv</b> thereafter. + * + * If the timer is already enabled, this function does nothing. + */ +void +periodic_timer_launch(periodic_timer_t *timer, const struct timeval *tv) +{ + tor_assert(timer); + if (event_pending(timer->ev, EV_TIMEOUT, NULL)) + return; + event_add(timer->ev, tv); +} + +/** + * Disable the provided <b>timer</b>, but do not free it. + * + * You can reenable the same timer later with periodic_timer_launch. + * + * If the timer is already disabled, this function does nothing. + */ +void +periodic_timer_disable(periodic_timer_t *timer) +{ + tor_assert(timer); + (void) event_del(timer->ev); +} + /** Stop and free a periodic timer */ void -periodic_timer_free(periodic_timer_t *timer) +periodic_timer_free_(periodic_timer_t *timer) { if (!timer) return; @@ -221,6 +296,173 @@ periodic_timer_free(periodic_timer_t *timer) tor_free(timer); } +/** + * Type used to represent events that run directly from the main loop, + * either because they are activated from elsewhere in the code, or + * because they have a simple timeout. + * + * We use this type to avoid exposing Libevent's API throughout the rest + * of the codebase. + * + * This type can't be used for all events: it doesn't handle events that + * are triggered by signals or by sockets. + */ +struct mainloop_event_t { + struct event *ev; + void (*cb)(mainloop_event_t *, void *); + void *userdata; +}; + +/** + * Internal: Implements mainloop event using a libevent event. + */ +static void +mainloop_event_cb(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + mainloop_event_t *mev = arg; + mev->cb(mev, mev->userdata); +} + +/** + * As mainloop_event_cb, but implements a post-loop event. + */ +static void +mainloop_event_postloop_cb(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + + /* Note that if rescan_mainloop_ev is already activated, + * event_active() will do nothing: only the first post-loop event that + * happens each time through the event loop will cause it to be + * activated. + * + * Because event_active() puts events on a FIFO queue, every event + * that is made active _after_ rescan_mainloop_ev will get its + * callback run after rescan_mainloop_cb is called -- that is, on the + * next iteration of the loop. + */ + event_active(rescan_mainloop_ev, EV_READ, 1); + + mainloop_event_t *mev = arg; + mev->cb(mev, mev->userdata); +} + +/** + * Helper for mainloop_event_new() and mainloop_event_postloop_new(). + */ +static mainloop_event_t * +mainloop_event_new_impl(int postloop, + void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + tor_assert(cb); + + struct event_base *base = tor_libevent_get_base(); + mainloop_event_t *mev = tor_malloc_zero(sizeof(mainloop_event_t)); + mev->ev = tor_event_new(base, -1, 0, + postloop ? mainloop_event_postloop_cb : mainloop_event_cb, + mev); + tor_assert(mev->ev); + mev->cb = cb; + mev->userdata = userdata; + return mev; +} + +/** + * Create and return a new mainloop_event_t to run the function <b>cb</b>. + * + * When run, the callback function will be passed the mainloop_event_t + * and <b>userdata</b> as its arguments. The <b>userdata</b> pointer + * must remain valid for as long as the mainloop_event_t event exists: + * it is your responsibility to free it. + * + * The event is not scheduled by default: Use mainloop_event_activate() + * or mainloop_event_schedule() to make it run. + */ +mainloop_event_t * +mainloop_event_new(void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + return mainloop_event_new_impl(0, cb, userdata); +} + +/** + * As mainloop_event_new(), but create a post-loop event. + * + * A post-loop event behaves like any ordinary event, but any events + * that _it_ activates cannot run until Libevent has checked for other + * events at least once. + */ +mainloop_event_t * +mainloop_event_postloop_new(void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + return mainloop_event_new_impl(1, cb, userdata); +} + +/** + * Schedule <b>event</b> to run in the main loop, immediately. If it is + * not scheduled, it will run anyway. If it is already scheduled to run + * later, it will run now instead. This function will have no effect if + * the event is already scheduled to run. + * + * This function may only be called from the main thread. + */ +void +mainloop_event_activate(mainloop_event_t *event) +{ + tor_assert(event); + event_active(event->ev, EV_READ, 1); +} + +/** Schedule <b>event</b> to run in the main loop, after a delay of <b>tv</b>. + * + * If the event is scheduled for a different time, cancel it and run + * after this delay instead. If the event is currently pending to run + * <em>now</b>, has no effect. + * + * Do not call this function with <b>tv</b> == NULL -- use + * mainloop_event_activate() instead. + * + * This function may only be called from the main thread. + */ +int +mainloop_event_schedule(mainloop_event_t *event, const struct timeval *tv) +{ + tor_assert(event); + if (BUG(tv == NULL)) { + // LCOV_EXCL_START + mainloop_event_activate(event); + return 0; + // LCOV_EXCL_STOP + } + return event_add(event->ev, tv); +} + +/** Cancel <b>event</b> if it is currently active or pending. (Do nothing if + * the event is not currently active or pending.) */ +void +mainloop_event_cancel(mainloop_event_t *event) +{ + if (!event) + return; + (void) event_del(event->ev); +} + +/** Cancel <b>event</b> and release all storage associated with it. */ +void +mainloop_event_free_(mainloop_event_t *event) +{ + if (!event) + return; + tor_event_free(event->ev); + memset(event, 0xb8, sizeof(*event)); + tor_free(event); +} + int tor_init_libevent_rng(void) { @@ -237,49 +479,58 @@ tor_init_libevent_rng(void) return rv; } -#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) \ - && !defined(TOR_UNIT_TESTS) +/** + * Un-initialize libevent in preparation for an exit + */ void -tor_gettimeofday_cached(struct timeval *tv) +tor_libevent_free_all(void) { - event_base_gettimeofday_cached(the_event_base, tv); + tor_event_free(rescan_mainloop_ev); + if (the_event_base) + event_base_free(the_event_base); + the_event_base = NULL; } -void -tor_gettimeofday_cache_clear(void) + +/** + * Run the event loop for the provided event_base, handling events until + * something stops it. If <b>once</b> is set, then just poll-and-run + * once, then exit. Return 0 on success, -1 if an error occurred, or 1 + * if we exited because no events were pending or active. + * + * This isn't reentrant or multithreaded. + */ +int +tor_libevent_run_event_loop(struct event_base *base, int once) { - event_base_update_cache_time(the_event_base); + const int flags = once ? EVLOOP_ONCE : 0; + return event_base_loop(base, flags); } -#else -/** Cache the current hi-res time; the cache gets reset when libevent - * calls us. */ -static struct timeval cached_time_hires = {0, 0}; -/** Return a fairly recent view of the current time. */ +/** Tell the event loop to exit after <b>delay</b>. If <b>delay</b> is NULL, + * instead exit after we're done running the currently active events. */ void -tor_gettimeofday_cached(struct timeval *tv) +tor_libevent_exit_loop_after_delay(struct event_base *base, + const struct timeval *delay) { - if (cached_time_hires.tv_sec == 0) { - tor_gettimeofday(&cached_time_hires); - } - *tv = cached_time_hires; + event_base_loopexit(base, delay); } -/** Reset the cached view of the current time, so that the next time we try - * to learn it, we will get an up-to-date value. */ +/** Tell the event loop to exit after running whichever callback is currently + * active. */ void -tor_gettimeofday_cache_clear(void) +tor_libevent_exit_loop_after_callback(struct event_base *base) { - cached_time_hires.tv_sec = 0; + event_base_loopbreak(base); } -#ifdef TOR_UNIT_TESTS -/** For testing: force-update the cached time to a given value. */ +#if defined(TOR_UNIT_TESTS) +/** For testing: called post-fork to make libevent reinitialize + * kernel structures. */ void -tor_gettimeofday_cache_set(const struct timeval *tv) +tor_libevent_postfork(void) { - tor_assert(tv); - memcpy(&cached_time_hires, tv, sizeof(*tv)); + int r = event_reinit(tor_libevent_get_base()); + tor_assert(r == 0); } -#endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index c2e34764e4..286a268122 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_LIBEVENT_H @@ -7,8 +7,6 @@ #include "orconfig.h" #include "testsupport.h" -#include <event2/event.h> - void configure_libevent_logging(void); void suppress_libevent_log_msg(const char *msg); @@ -19,7 +17,12 @@ void suppress_libevent_log_msg(const char *msg); evdns_add_server_port_with_base(tor_libevent_get_base(), \ (sock),(tcp),(cb),(data)); -void tor_event_free(struct event *ev); +struct event; +struct event_base; + +void tor_event_free_(struct event *ev); +#define tor_event_free(ev) \ + FREE_AND_NULL(struct event, tor_event_free_, (ev)) typedef struct periodic_timer_t periodic_timer_t; @@ -27,9 +30,25 @@ periodic_timer_t *periodic_timer_new(struct event_base *base, const struct timeval *tv, void (*cb)(periodic_timer_t *timer, void *data), void *data); -void periodic_timer_free(periodic_timer_t *); - -#define tor_event_base_loopexit event_base_loopexit +void periodic_timer_free_(periodic_timer_t *); +void periodic_timer_launch(periodic_timer_t *, const struct timeval *tv); +void periodic_timer_disable(periodic_timer_t *); +#define periodic_timer_free(t) \ + FREE_AND_NULL(periodic_timer_t, periodic_timer_free_, (t)) + +typedef struct mainloop_event_t mainloop_event_t; +mainloop_event_t *mainloop_event_new(void (*cb)(mainloop_event_t *, void *), + void *userdata); +mainloop_event_t * mainloop_event_postloop_new( + void (*cb)(mainloop_event_t *, void *), + void *userdata); +void mainloop_event_activate(mainloop_event_t *event); +int mainloop_event_schedule(mainloop_event_t *event, + const struct timeval *delay); +void mainloop_event_cancel(mainloop_event_t *event); +void mainloop_event_free_(mainloop_event_t *event); +#define mainloop_event_free(event) \ + FREE_AND_NULL(mainloop_event_t, mainloop_event_free_, (event)) /** Defines a configuration for using libevent with Tor: passed as an argument * to tor_libevent_initialize() to describe how we want to set up. */ @@ -47,15 +66,19 @@ const char *tor_libevent_get_method(void); void tor_check_libevent_header_compatibility(void); const char *tor_libevent_get_version_str(void); const char *tor_libevent_get_header_version_str(void); +void tor_libevent_free_all(void); int tor_init_libevent_rng(void); -void tor_gettimeofday_cached(struct timeval *tv); -void tor_gettimeofday_cache_clear(void); #ifdef TOR_UNIT_TESTS -void tor_gettimeofday_cache_set(const struct timeval *tv); +void tor_libevent_postfork(void); #endif +int tor_libevent_run_event_loop(struct event_base *base, int once); +void tor_libevent_exit_loop_after_delay(struct event_base *base, + const struct timeval *delay); +void tor_libevent_exit_loop_after_callback(struct event_base *base); + #ifdef COMPAT_LIBEVENT_PRIVATE /** Macro: returns the number of a Libevent version as a 4-byte number, @@ -69,7 +92,7 @@ void tor_gettimeofday_cache_set(const struct timeval *tv); STATIC void libevent_logging_callback(int severity, const char *msg); -#endif +#endif /* defined(COMPAT_LIBEVENT_PRIVATE) */ -#endif +#endif /* !defined(TOR_COMPAT_LIBEVENT_H) */ diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h index 1bfe188075..d1481fb46c 100644 --- a/src/common/compat_openssl.h +++ b/src/common/compat_openssl.h @@ -1,18 +1,19 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_OPENSSL_H #define TOR_COMPAT_OPENSSL_H #include <openssl/opensslv.h> +#include "crypto_openssl_mgt.h" /** * \file compat_openssl.h * - * \brief compatability definitions for working with different openssl forks + * \brief compatibility definitions for working with different openssl forks **/ #if !defined(LIBRESSL_VERSION_NUMBER) && \ @@ -25,10 +26,13 @@ /* We define this macro if we're trying to build with the majorly refactored * API in OpenSSL 1.1 */ #define OPENSSL_1_1_API +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) && ... */ + +#ifndef OPENSSL_VERSION +#define OPENSSL_VERSION SSLEAY_VERSION #endif #ifndef OPENSSL_1_1_API -#define OPENSSL_VERSION SSLEAY_VERSION #define OpenSSL_version(v) SSLeay_version(v) #define OpenSSL_version_num() SSLeay() #define RAND_OpenSSL() RAND_SSLeay() @@ -37,11 +41,11 @@ ((st) == SSL3_ST_SW_SRVR_HELLO_B)) #define OSSL_HANDSHAKE_STATE int #define CONST_IF_OPENSSL_1_1_API -#else +#else /* !(!defined(OPENSSL_1_1_API)) */ #define STATE_IS_SW_SERVER_HELLO(st) \ ((st) == TLS_ST_SW_SRVR_HELLO) #define CONST_IF_OPENSSL_1_1_API const -#endif +#endif /* !defined(OPENSSL_1_1_API) */ -#endif +#endif /* !defined(TOR_COMPAT_OPENSSL_H) */ diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c index c1ae66c1d2..002274c469 100644 --- a/src/common/compat_pthreads.c +++ b/src/common/compat_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -201,20 +201,21 @@ tor_cond_init(tor_cond_t *cond) } #if defined(HAVE_CLOCK_GETTIME) -#if defined(CLOCK_MONOTONIC) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) +#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \ + defined(CLOCK_MONOTONIC) /* Use monotonic time so when we timedwait() on it, any clock adjustment * won't affect the timeout value. */ if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) { return -1; } #define USE_COND_CLOCK CLOCK_MONOTONIC -#else /* !defined HAVE_PTHREAD_CONDATTR_SETCLOCK */ +#else /* !(defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ...) */ /* On OSX Sierra, there is no pthread_condattr_setclock, so we are stuck * with the realtime clock. */ #define USE_COND_CLOCK CLOCK_REALTIME -#endif /* which clock to use */ -#endif /* HAVE_CLOCK_GETTIME */ +#endif /* defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ... */ +#endif /* defined(HAVE_CLOCK_GETTIME) */ if (pthread_cond_init(&cond->cond, &condattr)) { return -1; } @@ -266,11 +267,11 @@ tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, const struct timeval *tv) tvnow.tv_sec = ts.tv_sec; tvnow.tv_usec = (int)(ts.tv_nsec / 1000); timeradd(tv, &tvnow, &tvsum); -#else +#else /* !(defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK)) */ if (gettimeofday(&tvnow, NULL) < 0) return -1; timeradd(tv, &tvnow, &tvsum); -#endif /* HAVE_CLOCK_GETTIME, CLOCK_MONOTONIC */ +#endif /* defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK) */ ts.tv_sec = tvsum.tv_sec; ts.tv_nsec = tvsum.tv_usec * 1000; diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index f4809060d6..9f64c06342 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -48,7 +48,7 @@ tor_mutex_new_nonrecursive(void) } /** Release all storage and system resources held by <b>m</b>. */ void -tor_mutex_free(tor_mutex_t *m) +tor_mutex_free_(tor_mutex_t *m) { if (!m) return; @@ -68,7 +68,7 @@ tor_cond_new(void) /** Free all storage held in <b>c</b>. */ void -tor_cond_free(tor_cond_t *c) +tor_cond_free_(tor_cond_t *c) { if (!c) return; @@ -94,51 +94,73 @@ in_main_thread(void) } #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE) -/* As write(), but retry on EINTR */ +/* As write(), but retry on EINTR, and return the negative error code on + * error. */ static int write_ni(int fd, const void *buf, size_t n) { int r; again: r = (int) write(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } -/* As read(), but retry on EINTR */ +/* As read(), but retry on EINTR, and return the negative error code on error. + */ static int read_ni(int fd, void *buf, size_t n) { int r; again: r = (int) read(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } -#endif +#endif /* defined(HAVE_EVENTFD) || defined(HAVE_PIPE) */ -/** As send(), but retry on EINTR. */ +/** As send(), but retry on EINTR, and return the negative error code on + * error. */ static int send_ni(int fd, const void *buf, size_t n, int flags) { int r; again: r = (int) send(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } -/** As recv(), but retry on EINTR. */ +/** As recv(), but retry on EINTR, and return the negative error code on + * error. */ static int recv_ni(int fd, void *buf, size_t n, int flags) { int r; again: r = (int) recv(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } @@ -149,7 +171,7 @@ eventfd_alert(int fd) { uint64_t u = 1; int r = write_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) + if (r < 0 && -r != EAGAIN) return -1; return 0; } @@ -160,11 +182,11 @@ eventfd_drain(int fd) { uint64_t u = 0; int r = read_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return r; return 0; } -#endif +#endif /* defined(HAVE_EVENTFD) */ #ifdef HAVE_PIPE /** Send a byte over a pipe. Return 0 on success or EAGAIN; -1 on error */ @@ -172,8 +194,8 @@ static int pipe_alert(int fd) { ssize_t r = write_ni(fd, "x", 1); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return (int)r; return 0; } @@ -188,11 +210,11 @@ pipe_drain(int fd) r = read_ni(fd, buf, sizeof(buf)); } while (r > 0); if (r < 0 && errno != EAGAIN) - return -1; + return -errno; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } -#endif +#endif /* defined(HAVE_PIPE) */ /** Send a byte on socket <b>fd</b>t. Return 0 on success or EAGAIN, * -1 on error. */ @@ -200,13 +222,13 @@ static int sock_alert(tor_socket_t fd) { ssize_t r = send_ni(fd, "x", 1, 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; return 0; } /** Drain all the input from a socket <b>fd</b>, and ignore it. Return 0 on - * success, -1 on error. */ + * success, -errno on error. */ static int sock_drain(tor_socket_t fd) { @@ -215,8 +237,8 @@ sock_drain(tor_socket_t fd) do { r = recv_ni(fd, buf, sizeof(buf), 0); } while (r > 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } @@ -254,7 +276,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) socks_out->drain_fn = eventfd_drain; return 0; } -#endif +#endif /* defined(HAVE_EVENTFD) */ #ifdef HAVE_PIPE2 /* Now we're going to try pipes. First type the pipe2() syscall, if we @@ -267,7 +289,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) socks_out->drain_fn = pipe_drain; return 0; } -#endif +#endif /* defined(HAVE_PIPE2) */ #ifdef HAVE_PIPE /* Now try the regular pipe() syscall. Pipes have a bit lower overhead than @@ -291,7 +313,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) socks_out->drain_fn = pipe_drain; return 0; } -#endif +#endif /* defined(HAVE_PIPE) */ /* If nothing else worked, fall back on socketpair(). */ if (!(flags & ASOCKS_NOSOCKETPAIR) && @@ -330,3 +352,55 @@ alert_sockets_close(alert_sockets_t *socks) socks->read_fd = socks->write_fd = -1; } +#ifndef HAVE_WORKING_STDATOMIC +/** Initialize a new atomic counter with the value 0 */ +void +atomic_counter_init(atomic_counter_t *counter) +{ + memset(counter, 0, sizeof(*counter)); + tor_mutex_init_nonrecursive(&counter->mutex); +} +/** Clean up all resources held by an atomic counter. */ +void +atomic_counter_destroy(atomic_counter_t *counter) +{ + tor_mutex_uninit(&counter->mutex); + memset(counter, 0, sizeof(*counter)); +} +/** Add a value to an atomic counter. */ +void +atomic_counter_add(atomic_counter_t *counter, size_t add) +{ + tor_mutex_acquire(&counter->mutex); + counter->val += add; + tor_mutex_release(&counter->mutex); +} +/** Subtract a value from an atomic counter. */ +void +atomic_counter_sub(atomic_counter_t *counter, size_t sub) +{ + // this relies on unsigned overflow, but that's fine. + atomic_counter_add(counter, -sub); +} +/** Return the current value of an atomic counter */ +size_t +atomic_counter_get(atomic_counter_t *counter) +{ + size_t val; + tor_mutex_acquire(&counter->mutex); + val = counter->val; + tor_mutex_release(&counter->mutex); + return val; +} +/** Replace the value of an atomic counter; return the old one. */ +size_t +atomic_counter_exchange(atomic_counter_t *counter, size_t newval) +{ + size_t oldval; + tor_mutex_acquire(&counter->mutex); + oldval = counter->val; + counter->val = newval; + tor_mutex_release(&counter->mutex); + return oldval; +} +#endif /* !defined(HAVE_WORKING_STDATOMIC) */ diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h index 171a9f93ff..8bf8225689 100644 --- a/src/common/compat_threads.h +++ b/src/common/compat_threads.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_THREADS_H @@ -14,13 +14,21 @@ #include <pthread.h> #endif +#if defined(HAVE_STDATOMIC_H) && defined(STDATOMIC_WORKS) +#define HAVE_WORKING_STDATOMIC +#endif + +#ifdef HAVE_WORKING_STDATOMIC +#include <stdatomic.h> +#endif + #if defined(_WIN32) #define USE_WIN32_THREADS #elif defined(HAVE_PTHREAD_H) && defined(HAVE_PTHREAD_CREATE) #define USE_PTHREADS #else #error "No threading system was found" -#endif +#endif /* defined(_WIN32) || ... */ int spawn_func(void (*func)(void *), void *data); void spawn_exit(void) ATTR_NORETURN; @@ -41,7 +49,7 @@ typedef struct tor_mutex_t { #else /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */ int _unused; -#endif +#endif /* defined(USE_WIN32_THREADS) || ... */ } tor_mutex_t; tor_mutex_t *tor_mutex_new(void); @@ -50,7 +58,8 @@ void tor_mutex_init(tor_mutex_t *m); void tor_mutex_init_nonrecursive(tor_mutex_t *m); void tor_mutex_acquire(tor_mutex_t *m); void tor_mutex_release(tor_mutex_t *m); -void tor_mutex_free(tor_mutex_t *m); +void tor_mutex_free_(tor_mutex_t *m); +#define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m)) void tor_mutex_uninit(tor_mutex_t *m); unsigned long tor_get_thread_id(void); void tor_threads_init(void); @@ -73,11 +82,12 @@ typedef struct tor_cond_t { int generation; #else #error no known condition implementation. -#endif +#endif /* defined(USE_PTHREADS) || ... */ } tor_cond_t; tor_cond_t *tor_cond_new(void); -void tor_cond_free(tor_cond_t *cond); +void tor_cond_free_(tor_cond_t *cond); +#define tor_cond_free(c) FREE_AND_NULL(tor_cond_t, tor_cond_free_, (c)) int tor_cond_init(tor_cond_t *cond); void tor_cond_uninit(tor_cond_t *cond); int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, @@ -147,5 +157,70 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal); */ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value); -#endif - +/** + * Atomic counter type; holds a size_t value. + */ +#ifdef HAVE_WORKING_STDATOMIC +typedef struct atomic_counter_t { + atomic_size_t val; +} atomic_counter_t; +#define ATOMIC_LINKAGE static +#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +typedef struct atomic_counter_t { + tor_mutex_t mutex; + size_t val; +} atomic_counter_t; +#define ATOMIC_LINKAGE +#endif /* defined(HAVE_WORKING_STDATOMIC) */ + +ATOMIC_LINKAGE void atomic_counter_init(atomic_counter_t *counter); +ATOMIC_LINKAGE void atomic_counter_destroy(atomic_counter_t *counter); +ATOMIC_LINKAGE void atomic_counter_add(atomic_counter_t *counter, size_t add); +ATOMIC_LINKAGE void atomic_counter_sub(atomic_counter_t *counter, size_t sub); +ATOMIC_LINKAGE size_t atomic_counter_get(atomic_counter_t *counter); +ATOMIC_LINKAGE size_t atomic_counter_exchange(atomic_counter_t *counter, + size_t newval); +#undef ATOMIC_LINKAGE + +#ifdef HAVE_WORKING_STDATOMIC +/** Initialize a new atomic counter with the value 0 */ +static inline void +atomic_counter_init(atomic_counter_t *counter) +{ + atomic_init(&counter->val, 0); +} +/** Clean up all resources held by an atomic counter. */ +static inline void +atomic_counter_destroy(atomic_counter_t *counter) +{ + (void)counter; +} +/** Add a value to an atomic counter. */ +static inline void +atomic_counter_add(atomic_counter_t *counter, size_t add) +{ + (void) atomic_fetch_add(&counter->val, add); +} +/** Subtract a value from an atomic counter. */ +static inline void +atomic_counter_sub(atomic_counter_t *counter, size_t sub) +{ + (void) atomic_fetch_sub(&counter->val, sub); +} +/** Return the current value of an atomic counter */ +static inline size_t +atomic_counter_get(atomic_counter_t *counter) +{ + return atomic_load(&counter->val); +} +/** Replace the value of an atomic counter; return the old one. */ +static inline size_t +atomic_counter_exchange(atomic_counter_t *counter, size_t newval) +{ + return atomic_exchange(&counter->val, newval); +} + +#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#endif /* defined(HAVE_WORKING_STDATOMIC) */ + +#endif /* !defined(TOR_COMPAT_THREADS_H) */ diff --git a/src/common/compat_time.c b/src/common/compat_time.c index d044bbe1d7..93b527def0 100644 --- a/src/common/compat_time.c +++ b/src/common/compat_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -28,7 +28,7 @@ /* as fallback implementation for tor_sleep_msec */ #include <sys/select.h> #endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ #ifdef __APPLE__ #include <mach/mach_time.h> @@ -64,15 +64,15 @@ tor_sleep_msec(int msec) select(0, NULL, NULL, NULL, &tv); #else sleep(CEIL_DIV(msec, 1000)); -#endif +#endif /* defined(_WIN32) || ... */ } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Set *timeval to the current time of day. On error, log and terminate. * (Same as gettimeofday(timeval,NULL), but never returns -1.) */ -void -tor_gettimeofday(struct timeval *timeval) +MOCK_IMPL(void, +tor_gettimeofday, (struct timeval *timeval)) { #ifdef _WIN32 /* Epoch bias copied from perl: number of units between windows epoch and @@ -90,7 +90,7 @@ tor_gettimeofday(struct timeval *timeval) if (ft.ft_64 < EPOCH_BIAS) { /* LCOV_EXCL_START */ log_err(LD_GENERAL,"System time is before 1970; failing."); - exit(1); + exit(1); // exit ok: system clock is broken. /* LCOV_EXCL_STOP */ } ft.ft_64 -= EPOCH_BIAS; @@ -102,7 +102,7 @@ tor_gettimeofday(struct timeval *timeval) log_err(LD_GENERAL,"gettimeofday failed."); /* If gettimeofday dies, we have either given a bad timezone (we didn't), or segfaulted.*/ - exit(1); + exit(1); // exit ok: gettimeofday failed. /* LCOV_EXCL_STOP */ } #elif defined(HAVE_FTIME) @@ -112,7 +112,7 @@ tor_gettimeofday(struct timeval *timeval) timeval->tv_usec = tb.millitm * 1000; #else #error "No way to get time." -#endif +#endif /* defined(_WIN32) || ... */ return; } @@ -187,8 +187,8 @@ monotime_coarse_set_mock_time_nsec(int64_t nsec) tor_assert_nonfatal(monotime_mocking_enabled == 1); mock_time_nsec_coarse = nsec; } -#endif -#endif +#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ +#endif /* defined(TOR_UNIT_TESTS) */ /* "ratchet" functions for monotonic time. */ @@ -235,7 +235,7 @@ ratchet_coarse_performance_counter(const int64_t count_raw) last_tick_count = count; return count; } -#endif +#endif /* defined(_WIN32) || defined(TOR_UNIT_TESTS) */ #if defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS) static struct timeval last_timeofday = { 0, 0 }; @@ -251,7 +251,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out) { /* must hold lock */ timeradd(timeval_raw, &timeofday_offset, out); - if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, <))) { + if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, OP_LT))) { /* time ran backwards. Instead, declare that no time occurred. */ timersub(&last_timeofday, timeval_raw, &timeofday_offset); memcpy(out, &last_timeofday, sizeof(struct timeval)); @@ -259,7 +259,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out) memcpy(&last_timeofday, out, sizeof(struct timeval)); } } -#endif +#endif /* defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS) */ #ifdef TOR_UNIT_TESTS /** For testing: reset all the ratchets */ @@ -271,7 +271,7 @@ monotime_reset_ratchets_for_testing(void) memset(&last_timeofday, 0, sizeof(struct timeval)); memset(&timeofday_offset, 0, sizeof(struct timeval)); } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ #ifdef __APPLE__ @@ -279,6 +279,9 @@ monotime_reset_ratchets_for_testing(void) * nanoseconds. */ static struct mach_timebase_info mach_time_info; +static struct mach_timebase_info mach_time_info_msec_cvt; +static int32_t mach_time_msec_cvt_threshold; +static int monotime_shift = 0; static void monotime_init_internal(void) @@ -287,6 +290,26 @@ monotime_init_internal(void) int r = mach_timebase_info(&mach_time_info); tor_assert(r == 0); tor_assert(mach_time_info.denom != 0); + + { + // approximate only. + uint64_t ns_per_tick = mach_time_info.numer / mach_time_info.denom; + uint64_t ms_per_tick = ns_per_tick * ONE_MILLION; + // requires that tor_log2(0) == 0. + monotime_shift = tor_log2(ms_per_tick); + } + { + // For converting ticks to milliseconds in a 32-bit-friendly way, we + // will first right-shift by 20, and then multiply by 2048/1953, since + // (1<<20) * 1953/2048 is about 1e6. We precompute a new numerator and + // denominator here to avoid multiple multiplies. + mach_time_info_msec_cvt.numer = mach_time_info.numer * 2048; + mach_time_info_msec_cvt.denom = mach_time_info.denom * 1953; + // For any value above this amount, we should divide before multiplying, + // to avoid overflow. For a value below this, we should multiply + // before dividing, to improve accuracy. + mach_time_msec_cvt_threshold = INT32_MAX / mach_time_info_msec_cvt.numer; + } } /** @@ -301,10 +324,25 @@ monotime_get(monotime_t *out) / mach_time_info.numer; return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ out->abstime_ = mach_absolute_time(); } +#if defined(HAVE_MACH_APPROXIMATE_TIME) +void +monotime_coarse_get(monotime_coarse_t *out) +{ +#ifdef TOR_UNIT_TESTS + if (monotime_mocking_enabled) { + out->abstime_ = (mock_time_nsec_coarse * mach_time_info.denom) + / mach_time_info.numer; + return; + } +#endif /* defined(TOR_UNIT_TESTS) */ + out->abstime_ = mach_approximate_time(); +} +#endif + /** * Return the number of nanoseconds between <b>start</b> and <b>end</b>. */ @@ -321,6 +359,47 @@ monotime_diff_nsec(const monotime_t *start, return diff_nsec; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + if (BUG(mach_time_info.denom == 0)) { + monotime_init(); + } + const int64_t diff_ticks = end->abstime_ - start->abstime_; + + /* We already require in di_ops.c that right-shift performs a sign-extend. */ + const int32_t diff_microticks = (int32_t)(diff_ticks >> 20); + + if (diff_microticks >= mach_time_msec_cvt_threshold) { + return (diff_microticks / mach_time_info_msec_cvt.denom) * + mach_time_info_msec_cvt.numer; + } else { + return (diff_microticks * mach_time_info_msec_cvt.numer) / + mach_time_info_msec_cvt.denom; + } +} + +uint32_t +monotime_coarse_to_stamp(const monotime_coarse_t *t) +{ + return (uint32_t)(t->abstime_ >> monotime_shift); +} + +int +monotime_is_zero(const monotime_t *val) +{ + return val->abstime_ == 0; +} + +void +monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec) +{ + const uint64_t nsec = msec * ONE_MILLION; + const uint64_t ticks = (nsec * mach_time_info.denom) / mach_time_info.numer; + out->abstime_ = val->abstime_ + ticks; +} + /* end of "__APPLE__" */ #elif defined(HAVE_CLOCK_GETTIME) @@ -332,7 +411,7 @@ monotime_diff_nsec(const monotime_t *start, * an old Linux kernel. In that case, we will fall back to CLOCK_MONOTONIC. */ static int clock_monotonic_coarse = CLOCK_MONOTONIC_COARSE; -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) */ static void monotime_init_internal(void) @@ -344,7 +423,7 @@ monotime_init_internal(void) "falling back to CLOCK_MONOTONIC.", strerror(errno)); clock_monotonic_coarse = CLOCK_MONOTONIC; } -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) */ } void @@ -356,7 +435,7 @@ monotime_get(monotime_t *out) out->ts_.tv_nsec = (int) (mock_time_nsec % ONE_BILLION); return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ int r = clock_gettime(CLOCK_MONOTONIC, &out->ts_); tor_assert(r == 0); } @@ -371,7 +450,7 @@ monotime_coarse_get(monotime_coarse_t *out) out->ts_.tv_nsec = (int) (mock_time_nsec_coarse % ONE_BILLION); return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ int r = clock_gettime(clock_monotonic_coarse, &out->ts_); if (PREDICT_UNLIKELY(r < 0) && errno == EINVAL && @@ -386,7 +465,7 @@ monotime_coarse_get(monotime_coarse_t *out) tor_assert(r == 0); } -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) */ int64_t monotime_diff_nsec(const monotime_t *start, @@ -399,6 +478,46 @@ monotime_diff_nsec(const monotime_t *start, return diff_nsec; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + const int32_t diff_sec = (int32_t)(end->ts_.tv_sec - start->ts_.tv_sec); + const int32_t diff_nsec = (int32_t)(end->ts_.tv_nsec - start->ts_.tv_nsec); + return diff_sec * 1000 + diff_nsec / ONE_MILLION; +} + +/* This value is ONE_BILLION >> 20. */ +static const uint32_t STAMP_TICKS_PER_SECOND = 953; + +uint32_t +monotime_coarse_to_stamp(const monotime_coarse_t *t) +{ + uint32_t nsec = (uint32_t)t->ts_.tv_nsec; + uint32_t sec = (uint32_t)t->ts_.tv_sec; + + return (sec * STAMP_TICKS_PER_SECOND) + (nsec >> 20); +} + +int +monotime_is_zero(const monotime_t *val) +{ + return val->ts_.tv_sec == 0 && val->ts_.tv_nsec == 0; +} + +void +monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec) +{ + const uint32_t sec = msec / 1000; + const uint32_t msec_remainder = msec % 1000; + out->ts_.tv_sec = val->ts_.tv_sec + sec; + out->ts_.tv_nsec = val->ts_.tv_nsec + (msec_remainder * ONE_MILLION); + if (out->ts_.tv_nsec > ONE_BILLION) { + out->ts_.tv_nsec -= ONE_BILLION; + out->ts_.tv_sec += 1; + } +} + /* end of "HAVE_CLOCK_GETTIME" */ #elif defined (_WIN32) @@ -462,7 +581,7 @@ monotime_get(monotime_t *out) / nsec_per_tick_numer; return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /* Alas, QueryPerformanceCounter is not always monotonic: see bug list at @@ -486,7 +605,7 @@ monotime_coarse_get(monotime_coarse_t *out) out->tick_count_ = mock_time_nsec_coarse / ONE_MILLION; return; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ if (GetTickCount64_fn) { out->tick_count_ = (int64_t)GetTickCount64_fn(); @@ -517,6 +636,13 @@ monotime_coarse_diff_msec(const monotime_coarse_t *start, return diff_ticks; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + return (int32_t)monotime_coarse_diff_msec(start, end); +} + int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end) @@ -531,6 +657,41 @@ monotime_coarse_diff_nsec(const monotime_coarse_t *start, return monotime_coarse_diff_msec(start, end) * ONE_MILLION; } +static const uint32_t STAMP_TICKS_PER_SECOND = 1000; + +uint32_t +monotime_coarse_to_stamp(const monotime_coarse_t *t) +{ + return (uint32_t) t->tick_count_; +} + +int +monotime_is_zero(const monotime_t *val) +{ + return val->pcount_ == 0; +} + +int +monotime_coarse_is_zero(const monotime_coarse_t *val) +{ + return val->tick_count_ == 0; +} + +void +monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec) +{ + const uint64_t nsec = msec * ONE_MILLION; + const uint64_t ticks = (nsec * nsec_per_tick_denom) / nsec_per_tick_numer; + out->pcount_ = val->pcount_ + ticks; +} + +void +monotime_coarse_add_msec(monotime_coarse_t *out, const monotime_coarse_t *val, + uint32_t msec) +{ + out->tick_count_ = val->tick_count_ + msec; +} + /* end of "_WIN32" */ #elif defined(MONOTIME_USING_GETTIMEOFDAY) @@ -567,10 +728,49 @@ monotime_diff_nsec(const monotime_t *start, return (diff.tv_sec * ONE_BILLION + diff.tv_usec * 1000); } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + struct timeval diff; + timersub(&end->tv_, &start->tv_, &diff); + return diff.tv_sec * 1000 + diff.tv_usec / 1000; +} + +/* This value is ONE_MILLION >> 10. */ +static const uint32_t STAMP_TICKS_PER_SECOND = 976; + +uint32_t +monotime_coarse_to_stamp(const monotime_coarse_t *t) +{ + const uint32_t usec = (uint32_t)t->tv_.tv_usec; + const uint32_t sec = (uint32_t)t->tv_.tv_sec; + return (sec * STAMP_TICKS_PER_SECOND) | (nsec >> 10); +} + +int +monotime_is_zero(const monotime_t *val) +{ + return val->tv_.tv_sec == 0 && val->tv_.tv_usec == 0; +} + +void +monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec) +{ + const uint32_t sec = msec / 1000; + const uint32_t msec_remainder = msec % 1000; + out->tv_.tv_sec = val->tv_.tv_sec + sec; + out->tv_.tv_usec = val->tv_.tv_nsec + (msec_remainder * 1000); + if (out->tv_.tv_usec > ONE_MILLION) { + out->tv_.tv_usec -= ONE_MILLION; + out->tv_.tv_sec += 1; + } +} + /* end of "MONOTIME_USING_GETTIMEOFDAY" */ #else #error "No way to implement monotonic timers." -#endif +#endif /* defined(__APPLE__) || ... */ /** * Initialize the monotonic timer subsystem. Must be called before any @@ -589,6 +789,19 @@ monotime_init(void) } } +void +monotime_zero(monotime_t *out) +{ + memset(out, 0, sizeof(*out)); +} +#ifdef MONOTIME_COARSE_TYPE_IS_DIFFERENT +void +monotime_coarse_zero(monotime_coarse_t *out) +{ + memset(out, 0, sizeof(*out)); +} +#endif + int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end) @@ -653,5 +866,48 @@ monotime_coarse_absolute_msec(void) { return monotime_coarse_absolute_nsec() / ONE_MILLION; } +#else +#define initialized_at_coarse initialized_at +#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ + +/** + * Return the current time "stamp" as described by monotime_coarse_to_stamp. + */ +uint32_t +monotime_coarse_get_stamp(void) +{ + monotime_coarse_t now; + monotime_coarse_get(&now); + return monotime_coarse_to_stamp(&now); +} + +#ifdef __APPLE__ +uint64_t +monotime_coarse_stamp_units_to_approx_msec(uint64_t units) +{ + /* Recover as much precision as we can. */ + uint64_t abstime_diff = (units << monotime_shift); + return (abstime_diff * mach_time_info.numer) / + (mach_time_info.denom * ONE_MILLION); +} +uint64_t +monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) +{ + uint64_t abstime_val = + (((uint64_t)msec) * ONE_MILLION * mach_time_info.denom) / + mach_time_info.numer; + return abstime_val >> monotime_shift; +} +#else +uint64_t +monotime_coarse_stamp_units_to_approx_msec(uint64_t units) +{ + return (units * 1000) / STAMP_TICKS_PER_SECOND; +} +uint64_t +monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) +{ + return (msec * STAMP_TICKS_PER_SECOND) / 1000; +} #endif diff --git a/src/common/compat_time.h b/src/common/compat_time.h index 2262446e57..f241aa5eba 100644 --- a/src/common/compat_time.h +++ b/src/common/compat_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -28,13 +28,13 @@ #include <time.h> #endif -#if !defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) +#if !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) /** Implementation of timeval for platforms that don't have it. */ struct timeval { time_t tv_sec; unsigned int tv_usec; }; -#endif +#endif /* !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) */ /** Represents a monotonic timer in a platform-dependent way. */ typedef struct monotime_t { @@ -51,10 +51,11 @@ typedef struct monotime_t { #define MONOTIME_USING_GETTIMEOFDAY /* Otherwise, we will be stuck using gettimeofday. */ struct timeval tv_; -#endif +#endif /* defined(__APPLE__) || ... */ } monotime_t; -#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC_COARSE) +#if defined(CLOCK_MONOTONIC_COARSE) && \ + defined(HAVE_CLOCK_GETTIME) #define MONOTIME_COARSE_FN_IS_DIFFERENT #define monotime_coarse_t monotime_t #elif defined(_WIN32) @@ -64,9 +65,12 @@ typedef struct monotime_t { typedef struct monotime_coarse_t { uint64_t tick_count_; } monotime_coarse_t; +#elif defined(__APPLE__) && defined(HAVE_MACH_APPROXIMATE_TIME) +#define MONOTIME_COARSE_FN_IS_DIFFERENT +#define monotime_coarse_t monotime_t #else #define monotime_coarse_t monotime_t -#endif +#endif /* defined(CLOCK_MONOTONIC_COARSE) && ... || ... */ /** * Initialize the timing subsystem. This function is idempotent. @@ -101,6 +105,21 @@ uint64_t monotime_absolute_usec(void); */ uint64_t monotime_absolute_msec(void); +/** + * Set <b>out</b> to zero. + */ +void monotime_zero(monotime_t *out); +/** + * Return true iff <b>out</b> is zero + */ +int monotime_is_zero(const monotime_t *out); + +/** + * Set <b>out</b> to N milliseconds after <b>val</b>. + */ +/* XXXX We should add a more generic function here if we ever need to */ +void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec); + #if defined(MONOTIME_COARSE_FN_IS_DIFFERENT) /** * Set <b>out</b> to the current coarse time. @@ -109,12 +128,30 @@ void monotime_coarse_get(monotime_coarse_t *out); uint64_t monotime_coarse_absolute_nsec(void); uint64_t monotime_coarse_absolute_usec(void); uint64_t monotime_coarse_absolute_msec(void); -#else +#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ #define monotime_coarse_get monotime_get #define monotime_coarse_absolute_nsec monotime_absolute_nsec #define monotime_coarse_absolute_usec monotime_absolute_usec #define monotime_coarse_absolute_msec monotime_absolute_msec -#endif +#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ + +/** + * Return a "timestamp" approximation for a coarse monotonic timer. + * This timestamp is meant to be fast to calculate and easy to + * compare, and have a unit of something roughly around 1 msec. + * + * It will wrap over from time to time. + * + * It has no defined zero point. + */ +uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t); +/** + * Convert a difference, expressed in the units of monotime_coarse_to_stamp, + * into an approximate number of milliseconds. + */ +uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units); +uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec); +uint32_t monotime_coarse_get_stamp(void); #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start, @@ -123,13 +160,48 @@ int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start, const monotime_coarse_t *end); -#else +void monotime_coarse_zero(monotime_coarse_t *out); +int monotime_coarse_is_zero(const monotime_coarse_t *val); +void monotime_coarse_add_msec(monotime_coarse_t *out, + const monotime_coarse_t *val, uint32_t msec); +#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */ #define monotime_coarse_diff_nsec monotime_diff_nsec #define monotime_coarse_diff_usec monotime_diff_usec #define monotime_coarse_diff_msec monotime_diff_msec +#define monotime_coarse_zero monotime_zero +#define monotime_coarse_is_zero monotime_is_zero +#define monotime_coarse_add_msec monotime_add_msec +#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ + +/** + * As monotime_coarse_diff_msec, but avoid 64-bit division. + * + * Requires that the difference fit into an int32_t; not for use with + * large time differences. + */ +int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end); + +/** + * As monotime_coarse_diff_msec, but avoid 64-bit division if it is expensive. + * + * Requires that the difference fit into an int32_t; not for use with + * large time differences. + */ +static inline int32_t +monotime_coarse_diff_msec32(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ +#if SIZEOF_VOID_P == 8 + // on a 64-bit platform, let's assume 64/64 division is cheap. + return (int32_t) monotime_coarse_diff_msec(start, end); +#else +#define USING_32BIT_MSEC_HACK + return monotime_coarse_diff_msec32_(start, end); #endif +} -void tor_gettimeofday(struct timeval *timeval); +MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval)); #ifdef TOR_UNIT_TESTS void tor_sleep_msec(int msec); @@ -142,7 +214,7 @@ void monotime_coarse_set_mock_time_nsec(int64_t); #else #define monotime_coarse_set_mock_time_nsec monotime_set_mock_time_nsec #endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ #ifdef COMPAT_TIME_PRIVATE #if defined(_WIN32) || defined(TOR_UNIT_TESTS) @@ -156,7 +228,7 @@ STATIC void ratchet_timeval(const struct timeval *timeval_raw, #ifdef TOR_UNIT_TESTS void monotime_reset_ratchets_for_testing(void); #endif -#endif +#endif /* defined(COMPAT_TIME_PRIVATE) */ -#endif +#endif /* !defined(TOR_COMPAT_TIME_H) */ diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c index 735be4ad17..7021344f6e 100644 --- a/src/common/compat_winthreads.c +++ b/src/common/compat_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -18,7 +18,6 @@ #include "util.h" #include "container.h" #include "torlog.h" -#include <process.h> /* This value is more or less total cargo-cult */ #define SPIN_COUNT 2000 @@ -48,10 +47,12 @@ void spawn_exit(void) { _endthread(); + // LCOV_EXCL_START //we should never get here. my compiler thinks that _endthread returns, this //is an attempt to fool it. tor_assert(0); - _exit(0); + _exit(0); // exit ok: unreachable. + // LCOV_EXCL_STOP } void @@ -246,5 +247,5 @@ tor_threads_init(void) set_main_thread(); } -#endif +#endif /* defined(_WIN32) */ diff --git a/src/common/compress.c b/src/common/compress.c new file mode 100644 index 0000000000..cb1549f1aa --- /dev/null +++ b/src/common/compress.c @@ -0,0 +1,675 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.c + * \brief Common compression API. + **/ + +#include "orconfig.h" + +#include <stdlib.h> +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include "torint.h" + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" +#include "compress_none.h" +#include "compress_zlib.h" +#include "compress_zstd.h" + +/** Total number of bytes allocated for compression state overhead. */ +static atomic_counter_t total_compress_allocation; + +/** @{ */ +/* These macros define the maximum allowable compression factor. Anything of + * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to + * have an uncompression factor (uncompressed size:compressed size ratio) of + * any greater than MAX_UNCOMPRESSION_FACTOR. + * + * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to + * be small to limit the attack multiplier, but we also want it to be large + * enough so that no legitimate document --even ones we might invent in the + * future -- ever compresses by a factor of greater than + * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably + * large range of possible values. IMO, anything over 8 is probably safe; IMO + * anything under 50 is probably sufficient. + */ +#define MAX_UNCOMPRESSION_FACTOR 25 +#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ + +/** Return true if uncompressing an input of size <b>in_size</b> to an input of + * size at least <b>size_out</b> looks like a compression bomb. */ +MOCK_IMPL(int, +tor_compress_is_compression_bomb,(size_t size_in, size_t size_out)) +{ + if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) + return 0; + + return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); +} + +/** Guess the size that <b>in_len</b> will be after compression or + * decompression. */ +static size_t +guess_compress_size(int compress, compress_method_t method, + compression_level_t compression_level, + size_t in_len) +{ + // ignore these for now. + (void)compression_level; + if (method == NO_METHOD) { + /* Guess that we'll need an extra byte, to avoid a needless realloc + * for nul-termination */ + return (in_len < SIZE_MAX) ? in_len + 1 : in_len; + } + + /* Always guess a factor of 2. */ + if (compress) { + in_len /= 2; + } else { + if (in_len < SIZE_T_CEILING/2) + in_len *= 2; + } + return MAX(in_len, 1024); +} + +/** Internal function to implement tor_compress/tor_uncompress, depending on + * whether <b>compress</b> is set. All arguments are as for tor_compress or + * tor_uncompress. */ +static int +tor_compress_impl(int compress, + char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + compression_level_t compression_level, + int complete_only, + int protocol_warn_level) +{ + tor_compress_state_t *stream; + int rv; + + stream = tor_compress_new(compress, method, compression_level); + + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + return -1; + } + + size_t in_len_orig = in_len; + size_t out_remaining, out_alloc; + char *outptr; + + out_remaining = out_alloc = + guess_compress_size(compress, method, compression_level, in_len); + *out = outptr = tor_malloc(out_remaining); + + const int finish = complete_only || compress; + + while (1) { + switch (tor_compress_process(stream, + &outptr, &out_remaining, + &in, &in_len, finish)) { + case TOR_COMPRESS_DONE: + if (in_len == 0 || compress) { + goto done; + } else { + // More data is present, and we're decompressing. So we may need to + // reinitialize the stream if we are handling multiple concatenated + // inputs. + tor_compress_free(stream); + stream = tor_compress_new(compress, method, compression_level); + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + goto err; + } + } + break; + case TOR_COMPRESS_OK: + if (compress || complete_only) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "Unexpected %s while %scompressing", + complete_only?"end of input":"result", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + goto err; + } else { + if (in_len == 0) { + goto done; + } + } + break; + case TOR_COMPRESS_BUFFER_FULL: { + if (!compress && outptr < *out+out_alloc) { + // A buffer error in this case means that we have a problem + // with our input. + log_fn(protocol_warn_level, LD_PROTOCOL, + "Possible truncated or corrupt compressed data"); + goto err; + } + if (out_alloc >= SIZE_T_CEILING / 2) { + log_warn(LD_GENERAL, "While %scompressing data: ran out of space.", + compress?"":"un"); + goto err; + } + if (!compress && + tor_compress_is_compression_bomb(in_len_orig, out_alloc)) { + // This should already have been caught down in the backend logic. + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + const size_t offset = outptr - *out; + out_alloc *= 2; + *out = tor_realloc(*out, out_alloc); + outptr = *out + offset; + out_remaining = out_alloc - offset; + break; + } + case TOR_COMPRESS_ERROR: + log_fn(protocol_warn_level, LD_GENERAL, + "Error while %scompressing data: bad input?", + compress?"":"un"); + goto err; // bad data. + + // LCOV_EXCL_START + default: + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + } + done: + *out_len = outptr - *out; + if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a " + "compression bomb."); + goto err; + } + if (!compress) { + // NUL-terminate our output. + if (out_alloc == *out_len) + *out = tor_realloc(*out, out_alloc + 1); + (*out)[*out_len] = '\0'; + } + rv = 0; + goto out; + + err: + tor_free(*out); + *out_len = 0; + rv = -1; + goto out; + + out: + tor_compress_free(stream); + return rv; +} + +/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly + * allocated buffer, using the method described in <b>method</b>. Store the + * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. + * Return 0 on success, -1 on failure. + */ +int +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + return tor_compress_impl(1, out, out_len, in, in_len, method, + BEST_COMPRESSION, + 1, LOG_WARN); +} + +/** Given zero or more compressed strings of total length <b>in_len</b> bytes + * at <b>in</b>, uncompress them into a newly allocated buffer, using the + * method described in <b>method</b>. Store the uncompressed string in + * *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 on + * failure. + * + * If any bytes are written to <b>out</b>, an extra byte NUL is always + * written at the end, but not counted in <b>out_len</b>. This is a + * safety feature to ensure that the output can be treated as a + * NUL-terminated string -- though of course, callers should check + * out_len anyway. + * + * If <b>complete_only</b> is true, we consider a truncated input as a + * failure; otherwise we decompress as much as we can. Warn about truncated + * or corrupt inputs at <b>protocol_warn_level</b>. + */ +int +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + return tor_compress_impl(0, out, out_len, in, in_len, method, + BEST_COMPRESSION, + complete_only, protocol_warn_level); +} + +/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely + * to be compressed or not. If it is, return the likeliest compression method. + * Otherwise, return UNKNOWN_METHOD. + */ +compress_method_t +detect_compression_method(const char *in, size_t in_len) +{ + if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { + return GZIP_METHOD; + } else if (in_len > 2 && (in[0] & 0x0f) == 8 && + (ntohs(get_uint16(in)) % 31) == 0) { + return ZLIB_METHOD; + } else if (in_len > 2 && + fast_memeq(in, "\x5d\x00\x00", 3)) { + return LZMA_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) { + return ZSTD_METHOD; + } else { + return UNKNOWN_METHOD; + } +} + +/** Return 1 if a given <b>method</b> is supported; otherwise 0. */ +int +tor_compress_supports_method(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_method_supported(); + case LZMA_METHOD: + return tor_lzma_method_supported(); + case ZSTD_METHOD: + return tor_zstd_method_supported(); + case NO_METHOD: + return 1; + case UNKNOWN_METHOD: + default: + return 0; + } +} + +/** + * Return a bitmask of the supported compression types, where 1<<m is + * set in the bitmask if and only if compression with method <b>m</b> is + * supported. + */ +unsigned +tor_compress_get_supported_method_bitmask(void) +{ + static unsigned supported = 0; + if (supported == 0) { + compress_method_t m; + for (m = NO_METHOD; m <= UNKNOWN_METHOD; ++m) { + if (tor_compress_supports_method(m)) { + supported |= (1u << m); + } + } + } + return supported; +} + +/** Table of compression method names. These should have an "x-" prefix, + * if they are not listed in the IANA content coding registry. */ +static const struct { + const char *name; + compress_method_t method; +} compression_method_names[] = { + { "gzip", GZIP_METHOD }, + { "deflate", ZLIB_METHOD }, + // We call this "x-tor-lzma" rather than "x-lzma", because we impose a + // lower maximum memory usage on the decoding side. + { "x-tor-lzma", LZMA_METHOD }, + { "x-zstd" , ZSTD_METHOD }, + { "identity", NO_METHOD }, + + /* Later entries in this table are not canonical; these are recognized but + * not emitted. */ + { "x-gzip", GZIP_METHOD }, +}; + +/** Return the canonical string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (method == compression_method_names[i].method) + return compression_method_names[i].name; + } + return NULL; +} + +/** Table of compression human readable method names. */ +static const struct { + compress_method_t method; + const char *name; +} compression_method_human_names[] = { + { NO_METHOD, "uncompressed" }, + { GZIP_METHOD, "gzipped" }, + { ZLIB_METHOD, "deflated" }, + { LZMA_METHOD, "LZMA compressed" }, + { ZSTD_METHOD, "Zstandard compressed" }, + { UNKNOWN_METHOD, "unknown encoding" }, +}; + +/** Return a human readable string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_human_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) { + if (method == compression_method_human_names[i].method) + return compression_method_human_names[i].name; + } + return NULL; +} + +/** Return the compression method represented by the string <b>name</b>, or + * UNKNOWN_METHOD if the string isn't recognized. */ +compress_method_t +compression_method_get_by_name(const char *name) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (!strcmp(compression_method_names[i].name, name)) + return compression_method_names[i].method; + } + return UNKNOWN_METHOD; +} + +/** Return a string representation of the version of the library providing the + * compression method given in <b>method</b>. Returns NULL if <b>method</b> is + * unknown or unsupported. */ +const char * +tor_compress_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_version_str(); + case LZMA_METHOD: + return tor_lzma_get_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return a string representation of the version of the library, found at + * compile time, providing the compression method given in <b>method</b>. + * Returns NULL if <b>method</b> is unknown or unsupported. */ +const char * +tor_compress_header_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_header_version_str(); + case LZMA_METHOD: + return tor_lzma_get_header_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_header_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return the approximate number of bytes allocated for all + * supported compression schemas. */ +size_t +tor_compress_get_total_allocation(void) +{ + return atomic_counter_get(&total_compress_allocation) + + tor_zlib_get_total_allocation() + + tor_lzma_get_total_allocation() + + tor_zstd_get_total_allocation(); +} + +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { + compress_method_t method; /**< The compression method. */ + + union { + tor_zlib_compress_state_t *zlib_state; + tor_lzma_compress_state_t *lzma_state; + tor_zstd_compress_state_t *zstd_state; + } u; /**< Compression backend state. */ +}; + +/** Construct and return a tor_compress_state_t object using <b>method</b>. If + * <b>compress</b>, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress, compress_method_t method, + compression_level_t compression_level) +{ + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + break; + } + case LZMA_METHOD: { + tor_lzma_compress_state_t *lzma_state = + tor_lzma_compress_new(compress, method, compression_level); + + if (lzma_state == NULL) + goto err; + + state->u.lzma_state = lzma_state; + break; + } + case ZSTD_METHOD: { + tor_zstd_compress_state_t *zstd_state = + tor_zstd_compress_new(compress, method, compression_level); + + if (zstd_state == NULL) + goto err; + + state->u.zstd_state = zstd_state; + break; + } + case NO_METHOD: { + break; + } + case UNKNOWN_METHOD: + goto err; + } + + atomic_counter_add(&total_compress_allocation, + sizeof(tor_compress_state_t)); + return state; + + err: + tor_free(state); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + tor_assert(state != NULL); + const size_t in_len_orig = *in_len; + const size_t out_len_orig = *out_len; + tor_compress_output_t rv; + + if (*out_len == 0 && (*in_len > 0 || finish)) { + // If we still have input data, but no space for output data, we might as + // well return early and let the caller do the reallocation of the out + // variable. + return TOR_COMPRESS_BUFFER_FULL; + } + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + rv = tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + break; + case LZMA_METHOD: + rv = tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); + break; + case ZSTD_METHOD: + rv = tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); + break; + case NO_METHOD: + rv = tor_cnone_compress_process(out, out_len, in, in_len, + finish); + break; + default: + case UNKNOWN_METHOD: + goto err; + } + if (BUG((rv == TOR_COMPRESS_OK) && + *in_len == in_len_orig && + *out_len == out_len_orig)) { + log_warn(LD_GENERAL, + "More info on the bug: method == %s, finish == %d, " + " *in_len == in_len_orig == %lu, " + "*out_len == out_len_orig == %lu", + compression_method_get_human_name(state->method), finish, + (unsigned long)in_len_orig, (unsigned long)out_len_orig); + return TOR_COMPRESS_ERROR; + } + + return rv; + err: + return TOR_COMPRESS_ERROR; +} + +/** Deallocate <b>state</b>. */ +void +tor_compress_free_(tor_compress_state_t *state) +{ + if (state == NULL) + return; + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + tor_zlib_compress_free(state->u.zlib_state); + break; + case LZMA_METHOD: + tor_lzma_compress_free(state->u.lzma_state); + break; + case ZSTD_METHOD: + tor_zstd_compress_free(state->u.zstd_state); + break; + case NO_METHOD: + break; + case UNKNOWN_METHOD: + break; + } + + atomic_counter_sub(&total_compress_allocation, + sizeof(tor_compress_state_t)); + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_compress_state_size(const tor_compress_state_t *state) +{ + tor_assert(state != NULL); + + size_t size = sizeof(tor_compress_state_t); + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + size += tor_zlib_compress_state_size(state->u.zlib_state); + break; + case LZMA_METHOD: + size += tor_lzma_compress_state_size(state->u.lzma_state); + break; + case ZSTD_METHOD: + size += tor_zstd_compress_state_size(state->u.zstd_state); + break; + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } + + return size; +} + +/** Initialize all compression modules. */ +void +tor_compress_init(void) +{ + atomic_counter_init(&total_compress_allocation); + + tor_zlib_init(); + tor_lzma_init(); + tor_zstd_init(); +} + +/** Warn if we had any problems while setting up our compression libraries. + * + * (This isn't part of tor_compress_init, since the logs aren't set up yet.) + */ +void +tor_compress_log_init_warnings(void) +{ + tor_zstd_warn_if_version_mismatched(); +} + diff --git a/src/common/compress.h b/src/common/compress.h new file mode 100644 index 0000000000..65d63a4386 --- /dev/null +++ b/src/common/compress.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.h + * \brief Headers for compress.c + **/ + +#ifndef TOR_COMPRESS_H +#define TOR_COMPRESS_H + +/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and + * GZIP_METHOD is guaranteed to be supported by the compress/uncompress + * functions here. Call tor_compress_supports_method() to check if a given + * compression schema is supported by Tor. */ +typedef enum { + NO_METHOD=0, // This method must be first. + GZIP_METHOD=1, + ZLIB_METHOD=2, + LZMA_METHOD=3, + ZSTD_METHOD=4, + UNKNOWN_METHOD=5, // This method must be last. Add new ones in the middle. +} compress_method_t; + +/** + * Enumeration to define tradeoffs between memory usage and compression level. + * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most + * memory. + **/ +typedef enum { + BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION +} compression_level_t; + +int tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +compress_method_t detect_compression_method(const char *in, size_t in_len); + +MOCK_DECL(int,tor_compress_is_compression_bomb,(size_t size_in, + size_t size_out)); + +int tor_compress_supports_method(compress_method_t method); +unsigned tor_compress_get_supported_method_bitmask(void); +const char *compression_method_get_name(compress_method_t method); +const char *compression_method_get_human_name(compress_method_t method); +compress_method_t compression_method_get_by_name(const char *name); + +const char *tor_compress_version_str(compress_method_t method); + +const char *tor_compress_header_version_str(compress_method_t method); + +size_t tor_compress_get_total_allocation(void); + +/** Return values from tor_compress_process; see that function's documentation + * for details. */ +typedef enum { + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; + +/** Internal state for an incremental compression/decompression. */ +typedef struct tor_compress_state_t tor_compress_state_t; + +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); + +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free_(tor_compress_state_t *state); +#define tor_compress_free(st) \ + FREE_AND_NULL(tor_compress_state_t, tor_compress_free_, (st)) + +size_t tor_compress_state_size(const tor_compress_state_t *state); + +void tor_compress_init(void); +void tor_compress_log_init_warnings(void); + +#endif /* !defined(TOR_COMPRESS_H) */ + diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c new file mode 100644 index 0000000000..051c59ba2d --- /dev/null +++ b/src/common/compress_lzma.c @@ -0,0 +1,361 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.c + * \brief Compression backend for LZMA. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" + +#ifdef HAVE_LZMA +#include <lzma.h> +#endif + +/** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */ +#define MEMORY_LIMIT (16 * 1024 * 1024) + +/** Total number of bytes allocated for LZMA state. */ +static atomic_counter_t total_lzma_allocation; + +#ifdef HAVE_LZMA +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 6; + case MEDIUM_COMPRESSION: return 4; + case LOW_COMPRESSION: return 2; + } +} + +/** Convert a given <b>error</b> to a human readable error string. */ +static const char * +lzma_error_str(lzma_ret error) +{ + switch (error) { + case LZMA_OK: + return "Operation completed successfully"; + case LZMA_STREAM_END: + return "End of stream"; + case LZMA_NO_CHECK: + return "Input stream lacks integrity check"; + case LZMA_UNSUPPORTED_CHECK: + return "Unable to calculate integrity check"; + case LZMA_GET_CHECK: + return "Integrity check available"; + case LZMA_MEM_ERROR: + return "Unable to allocate memory"; + case LZMA_MEMLIMIT_ERROR: + return "Memory limit reached"; + case LZMA_FORMAT_ERROR: + return "Unknown file format"; + case LZMA_OPTIONS_ERROR: + return "Unsupported options"; + case LZMA_DATA_ERROR: + return "Corrupt input data"; + case LZMA_BUF_ERROR: + return "Unable to progress"; + case LZMA_PROG_ERROR: + return "Programming error"; + default: + return "Unknown LZMA error"; + } +} +#endif /* defined(HAVE_LZMA) */ + +/** Return 1 if LZMA compression is supported; otherwise 0. */ +int +tor_lzma_method_supported(void) +{ +#ifdef HAVE_LZMA + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of liblzma. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_version_str(void) +{ +#ifdef HAVE_LZMA + return lzma_version_string(); +#else + return NULL; +#endif +} + +/** Return a string representation of the version of liblzma used at + * compilation time. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_header_version_str(void) +{ +#ifdef HAVE_LZMA + return LZMA_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal LZMA state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_lzma_compress_state_t { +#ifdef HAVE_LZMA + lzma_stream stream; /**< The LZMA stream. */ +#endif + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_LZMA +/** Return an approximate number of bytes stored in memory to hold the LZMA + * encoder/decoder state. */ +static size_t +tor_lzma_state_size_precalc(int compress, compression_level_t level) +{ + uint64_t memory_usage; + + if (compress) + memory_usage = lzma_easy_encoder_memusage(memory_level(level)); + else + memory_usage = lzma_easy_decoder_memusage(memory_level(level)); + + if (memory_usage == UINT64_MAX) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s", + compress ? "encoder" : "decoder"); + goto err; + // LCOV_EXCL_STOP + } + + if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX) + memory_usage = SIZE_MAX; + else + memory_usage += sizeof(tor_lzma_compress_state_t); + + return (size_t)memory_usage; + + // LCOV_EXCL_START + err: + return 0; + // LCOV_EXCL_STOP +} +#endif /* defined(HAVE_LZMA) */ + +/** Construct and return a tor_lzma_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == LZMA_METHOD); + +#ifdef HAVE_LZMA + tor_lzma_compress_state_t *result; + lzma_ret retval; + lzma_options_lzma stream_options; + + // Note that we do not explicitly initialize the lzma_stream object here, + // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is + // also what `tor_malloc_zero()` does. + result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); + result->compress = compress; + result->allocation = tor_lzma_state_size_precalc(compress, level); + + if (compress) { + lzma_lzma_preset(&stream_options, memory_level(level)); + + retval = lzma_alone_encoder(&result->stream, &stream_options); + + if (retval != LZMA_OK) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + // LCOV_EXCL_STOP + } + } else { + retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT); + + if (retval != LZMA_OK) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + // LCOV_EXCL_STOP + } + } + + atomic_counter_add(&total_lzma_allocation, result->allocation); + return result; + + /* LCOV_EXCL_START */ + err: + tor_free(result); + return NULL; + /* LCOV_EXCL_STOP */ +#else /* !(defined(HAVE_LZMA)) */ + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif /* defined(HAVE_LZMA) */ +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_LZMA + lzma_ret retval; + lzma_action action; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + state->stream.next_in = (unsigned char *)*in; + state->stream.avail_in = *in_len; + state->stream.next_out = (unsigned char *)*out; + state->stream.avail_out = *out_len; + + action = finish ? LZMA_FINISH : LZMA_RUN; + + retval = lzma_code(&state->stream, action); + + state->input_so_far += state->stream.next_in - ((unsigned char *)*in); + state->output_so_far += state->stream.next_out - ((unsigned char *)*out); + + *out = (char *)state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *)state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (retval) { + case LZMA_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_OK; + + case LZMA_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + + return TOR_COMPRESS_BUFFER_FULL; + + case LZMA_STREAM_END: + return TOR_COMPRESS_DONE; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + lzma_error_str(retval)); + return TOR_COMPRESS_ERROR; + } +#else /* !(defined(HAVE_LZMA)) */ + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + return TOR_COMPRESS_ERROR; +#endif /* defined(HAVE_LZMA) */ +} + +/** Deallocate <b>state</b>. */ +void +tor_lzma_compress_free_(tor_lzma_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_lzma_allocation, state->allocation); + +#ifdef HAVE_LZMA + lzma_end(&state->stream); +#endif + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all LZMA states. */ +size_t +tor_lzma_get_total_allocation(void) +{ + return atomic_counter_get(&total_lzma_allocation); +} + +/** Initialize the lzma module */ +void +tor_lzma_init(void) +{ + atomic_counter_init(&total_lzma_allocation); +} + diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h new file mode 100644 index 0000000000..38a447c1f3 --- /dev/null +++ b/src/common/compress_lzma.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.h + * \brief Header for compress_lzma.c + **/ + +#ifndef TOR_COMPRESS_LZMA_H +#define TOR_COMPRESS_LZMA_H + +int tor_lzma_method_supported(void); + +const char *tor_lzma_get_version_str(void); + +const char *tor_lzma_get_header_version_str(void); + +/** Internal state for an incremental LZMA compression/decompression. */ +typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; + +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_lzma_compress_free_(tor_lzma_compress_state_t *state); +#define tor_lzma_compress_free(st) \ + FREE_AND_NULL(tor_lzma_compress_state_t, \ + tor_lzma_compress_free_, (st)) + +size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); + +size_t tor_lzma_get_total_allocation(void); + +void tor_lzma_init(void); + +#endif /* !defined(TOR_COMPRESS_LZMA_H) */ + diff --git a/src/common/compress_none.c b/src/common/compress_none.c new file mode 100644 index 0000000000..34314e4af7 --- /dev/null +++ b/src/common/compress_none.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.c + * \brief Compression backend for identity compression. + * + * We actually define this backend so that we can treat the identity transform + * as another case of compression. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_none.h" + +/** Transfer some bytes using the identity transformation. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + size_t n_to_copy = MIN(*in_len, *out_len); + + memcpy(*out, *in, n_to_copy); + *out += n_to_copy; + *in += n_to_copy; + *out_len -= n_to_copy; + *in_len -= n_to_copy; + if (*in_len == 0) { + return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK; + } else { + return TOR_COMPRESS_BUFFER_FULL; + } +} + diff --git a/src/common/compress_none.h b/src/common/compress_none.h new file mode 100644 index 0000000000..77c3cef47b --- /dev/null +++ b/src/common/compress_none.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.h + * \brief Header for compress_none.c + **/ + +#ifndef TOR_COMPRESS_NONE_H +#define TOR_COMPRESS_NONE_H + +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +#endif /* !defined(TOR_COMPRESS_NONE_H) */ + diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000000..23d71d27be --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,304 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include <zlib.h> + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static atomic_counter_t total_zlib_allocation; + +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: return 9; + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + +/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return 1 if zlib/gzip compression is supported; otherwise 0. */ +int +tor_zlib_method_supported(void) +{ + /* We currently always support zlib/gzip, but we keep this function around in + * case we some day decide to deprecate zlib/gzip support. + */ + return 1; +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits <b>windowBits</b> and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress_, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress_) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = BEST_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress_; + bits = method_bits(method, compression_level); + memlevel = memory_level(compression_level); + if (compress_) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); + + atomic_counter_add(&total_zlib_allocation, out->allocation); + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + if (*in_len > UINT_MAX || + *out_len > UINT_MAX) { + return TOR_COMPRESS_ERROR; + } + + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : "<no message>"); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate <b>state</b>. */ +void +tor_zlib_compress_free_(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zlib_allocation, state->allocation); + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return atomic_counter_get(&total_zlib_allocation); +} + +/** Set up global state for the zlib module */ +void +tor_zlib_init(void) +{ + atomic_counter_init(&total_zlib_allocation); +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000000..e3c1a2b339 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +int tor_zlib_method_supported(void); + +const char *tor_zlib_get_version_str(void); + +const char *tor_zlib_get_header_version_str(void); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zlib_compress_free_(tor_zlib_compress_state_t *state); +#define tor_zlib_compress_free(st) \ + FREE_AND_NULL(tor_zlib_compress_state_t, \ + tor_zlib_compress_free_, (st)) + +size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t tor_zlib_get_total_allocation(void); + +void tor_zlib_init(void); + +#endif /* !defined(TOR_COMPRESS_ZLIB_H) */ + diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c new file mode 100644 index 0000000000..dc8b4d621d --- /dev/null +++ b/src/common/compress_zstd.c @@ -0,0 +1,540 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.c + * \brief Compression backend for Zstandard. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zstd.h" + +#ifdef ENABLE_ZSTD_ADVANCED_APIS +/* This is a lie, but we make sure it doesn't get us in trouble by wrapping + * all invocations of zstd's static-only functions in a check to make sure + * that the compile-time version matches the run-time version. */ +#define ZSTD_STATIC_LINKING_ONLY +#endif + +#ifdef HAVE_ZSTD +#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE +DISABLE_GCC_WARNING(unused-const-variable) +#endif +#include <zstd.h> +#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE +ENABLE_GCC_WARNING(unused-const-variable) +#endif +#endif + +/** Total number of bytes allocated for Zstandard state. */ +static atomic_counter_t total_zstd_allocation; + +#ifdef HAVE_ZSTD +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 8; + case LOW_COMPRESSION: return 7; + } +} +#endif /* defined(HAVE_ZSTD) */ + +/** Return 1 if Zstandard compression is supported; otherwise 0. */ +int +tor_zstd_method_supported(void) +{ +#ifdef HAVE_ZSTD + return 1; +#else + return 0; +#endif +} + +#ifdef HAVE_ZSTD +/** Format a zstd version number as a string in <b>buf</b>. */ +static void +tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) +{ + tor_snprintf(buf, buflen, + "%u.%u.%u", + version_number / 10000 % 100, + version_number / 100 % 100, + version_number % 100); +} +#endif + +#define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */ + +/** Return a string representation of the version of the currently running + * version of libzstd. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_version_str(void) +{ +#ifdef HAVE_ZSTD + static char version_str[VERSION_STR_MAX_LEN]; + + tor_zstd_format_version(version_str, sizeof(version_str), + ZSTD_versionNumber()); + + return version_str; +#else /* !(defined(HAVE_ZSTD)) */ + return NULL; +#endif /* defined(HAVE_ZSTD) */ +} + +/** Return a string representation of the version of the version of libzstd + * used at compilation time. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_header_version_str(void) +{ +#ifdef HAVE_ZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} + +#ifdef TOR_UNIT_TESTS +static int static_apis_disable_for_testing = 0; +#endif + +/** Return true iff we can use the "static-only" APIs. */ +int +tor_zstd_can_use_static_apis(void) +{ +#if defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) +#ifdef TOR_UNIT_TESTS + if (static_apis_disable_for_testing) { + return 0; + } +#endif + return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber()); +#else + return 0; +#endif +} + +/** Internal Zstandard state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zstd_compress_state_t { +#ifdef HAVE_ZSTD + union { + /** Compression stream. Used when <b>compress</b> is true. */ + ZSTD_CStream *compress_stream; + /** Decompression stream. Used when <b>compress</b> is false. */ + ZSTD_DStream *decompress_stream; + } u; /**< Zstandard stream objects. */ +#endif /* defined(HAVE_ZSTD) */ + + int compress; /**< True if we are compressing; false if we are inflating */ + int have_called_end; /**< True if we are compressing and we've called + * ZSTD_endStream */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_ZSTD +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. This is a fake estimate + * based on inspecting the zstd source: tor_zstd_state_size_precalc() is + * more accurate when it's allowed to use "static-only" functions */ +static size_t +tor_zstd_state_size_precalc_fake(int compress, int preset) +{ + tor_assert(preset > 0); + + size_t memory_usage = sizeof(tor_zstd_compress_state_t); + + // The Zstandard library provides a number of functions that would be useful + // here, but they are, unfortunately, still considered experimental and are + // thus only available in libzstd if we link against the library statically. + // + // The code in this function tries to approximate the calculations without + // being able to use the following: + // + // - We do not have access to neither the internal members of ZSTD_CStream + // and ZSTD_DStream and their internal context objects. + // + // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they + // are unexposed. + // + // In the future it might be useful to check if libzstd have started + // providing these functions in a stable manner and simplify this function. + if (compress) { + // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine: + memory_usage += 192; + + // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to + // variables that are not exposed via the public API. We use a _very_ + // simplified function to calculate the estimated amount of bytes used in + // this struct. + // memory_usage += (preset - 0.5) * 1024 * 1024; + memory_usage += (preset * 1024 * 1024) - (512 * 1024); + // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes. + // - stream->outBuffSize: 128 KB: + memory_usage += 128 * 1024; + // - stream->inBuffSize: 2048 KB: + memory_usage += 2048 * 1024; + } else { + // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine: + memory_usage += 208; + // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB. + memory_usage += 150 * 1024; + + // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes. + // - stream->inBuffSize: 0 KB. + // - stream->outBuffSize: 0 KB. + } + + return memory_usage; +} + +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ +#ifdef ZSTD_STATIC_LINKING_ONLY + if (tor_zstd_can_use_static_apis()) { + if (compress) { +#ifdef HAVE_ZSTD_ESTIMATECSTREAMSIZE + return ZSTD_estimateCStreamSize(preset); +#endif + } else { +#ifdef HAVE_ZSTD_ESTIMATEDCTXSIZE + /* Could use DStream, but that takes a windowSize. */ + return ZSTD_estimateDCtxSize(); +#endif + } + } +#endif + return tor_zstd_state_size_precalc_fake(compress, preset); +} +#endif /* defined(HAVE_ZSTD) */ + +/** Construct and return a tor_zstd_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == ZSTD_METHOD); + +#ifdef HAVE_ZSTD + const int preset = memory_level(level); + tor_zstd_compress_state_t *result; + size_t retval; + + result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); + result->compress = compress; + result->allocation = tor_zstd_state_size_precalc(compress, preset); + + if (compress) { + result->u.compress_stream = ZSTD_createCStream(); + + if (result->u.compress_stream == NULL) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error while creating Zstandard compression " + "stream"); + goto err; + // LCOV_EXCL_STOP + } + + retval = ZSTD_initCStream(result->u.compress_stream, preset); + + if (ZSTD_isError(retval)) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + // LCOV_EXCL_STOP + } + } else { + result->u.decompress_stream = ZSTD_createDStream(); + + if (result->u.decompress_stream == NULL) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error while creating Zstandard decompression " + "stream"); + goto err; + // LCOV_EXCL_STOP + } + + retval = ZSTD_initDStream(result->u.decompress_stream); + + if (ZSTD_isError(retval)) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + // LCOV_EXCL_STOP + } + } + + atomic_counter_add(&total_zstd_allocation, result->allocation); + return result; + + err: + // LCOV_EXCL_START + if (compress) { + ZSTD_freeCStream(result->u.compress_stream); + } else { + ZSTD_freeDStream(result->u.decompress_stream); + } + + tor_free(result); + return NULL; + // LCOV_EXCL_STOP +#else /* !(defined(HAVE_ZSTD)) */ + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif /* defined(HAVE_ZSTD) */ +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_ZSTD + size_t retval; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + ZSTD_inBuffer input = { *in, *in_len, 0 }; + ZSTD_outBuffer output = { *out, *out_len, 0 }; + + if (BUG(finish == 0 && state->have_called_end)) { + finish = 1; + } + + if (state->compress) { + if (! state->have_called_end) + retval = ZSTD_compressStream(state->u.compress_stream, + &output, &input); + else + retval = 0; + } else { + retval = ZSTD_decompressStream(state->u.decompress_stream, + &output, &input); + } + + state->input_so_far += input.pos; + state->output_so_far += output.pos; + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + *in = (char *)input.src + input.pos; + *in_len = input.size - input.pos; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (state->compress && !state->have_called_end) { + retval = ZSTD_flushStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // ZSTD_flushStream returns 0 if the frame is done, or >0 if it + // is incomplete. + if (retval > 0) { + return TOR_COMPRESS_BUFFER_FULL; + } + } + + if (!finish) { + // The caller says we're not done with the input, so no need to write an + // epilogue. + return TOR_COMPRESS_OK; + } else if (state->compress) { + if (*in_len) { + // We say that we're not done with the input, so we can't write an + // epilogue. + return TOR_COMPRESS_OK; + } + + retval = ZSTD_endStream(state->u.compress_stream, &output); + state->have_called_end = 1; + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to write " + "epilogue: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // endStream returns the number of bytes that is needed to write the + // epilogue. + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_DONE; + } else /* if (!state->compress) */ { + // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it + // is incomplete. + // We check this above. + tor_assert_nonfatal(!ZSTD_isError(retval)); + // Start a new frame if this frame is done + if (retval == 0) + return TOR_COMPRESS_DONE; + // Don't check out_len, it might have some space left if the next output + // chunk is larger than the remaining space + else if (*in_len > 0) + return TOR_COMPRESS_BUFFER_FULL; + else + return TOR_COMPRESS_OK; + } + +#else /* !(defined(HAVE_ZSTD)) */ + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + + return TOR_COMPRESS_ERROR; +#endif /* defined(HAVE_ZSTD) */ +} + +/** Deallocate <b>state</b>. */ +void +tor_zstd_compress_free_(tor_zstd_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zstd_allocation, state->allocation); + +#ifdef HAVE_ZSTD + if (state->compress) { + ZSTD_freeCStream(state->u.compress_stream); + } else { + ZSTD_freeDStream(state->u.decompress_stream); + } +#endif /* defined(HAVE_ZSTD) */ + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all Zstandard + * states. */ +size_t +tor_zstd_get_total_allocation(void) +{ + return atomic_counter_get(&total_zstd_allocation); +} + +/** Initialize the zstd module */ +void +tor_zstd_init(void) +{ + atomic_counter_init(&total_zstd_allocation); +} + +/** Warn if the header and library versions don't match. */ +void +tor_zstd_warn_if_version_mismatched(void) +{ +#if defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) + if (! tor_zstd_can_use_static_apis()) { + char header_version[VERSION_STR_MAX_LEN]; + char runtime_version[VERSION_STR_MAX_LEN]; + tor_zstd_format_version(header_version, sizeof(header_version), + ZSTD_VERSION_NUMBER); + tor_zstd_format_version(runtime_version, sizeof(runtime_version), + ZSTD_versionNumber()); + + log_warn(LD_GENERAL, + "Tor was compiled with zstd %s, but is running with zstd %s. " + "For safety, we'll avoid using advanced zstd functionality.", + header_version, runtime_version); + } +#endif +} + +#ifdef TOR_UNIT_TESTS +/** Testing only: disable usage of static-only APIs, so we can make sure that + * we still work without them. */ +void +tor_zstd_set_static_apis_disabled_for_testing(int disabled) +{ + static_apis_disable_for_testing = disabled; +} +#endif + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h new file mode 100644 index 0000000000..bd42cf65ce --- /dev/null +++ b/src/common/compress_zstd.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.h + * \brief Header for compress_zstd.c + **/ + +#ifndef TOR_COMPRESS_ZSTD_H +#define TOR_COMPRESS_ZSTD_H + +int tor_zstd_method_supported(void); + +const char *tor_zstd_get_version_str(void); + +const char *tor_zstd_get_header_version_str(void); + +int tor_zstd_can_use_static_apis(void); + +/** Internal state for an incremental Zstandard compression/decompression. */ +typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; + +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zstd_compress_free_(tor_zstd_compress_state_t *state); +#define tor_zstd_compress_free(st) \ + FREE_AND_NULL(tor_zstd_compress_state_t, \ + tor_zstd_compress_free_, (st)) + +size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); + +size_t tor_zstd_get_total_allocation(void); + +void tor_zstd_init(void); +void tor_zstd_warn_if_version_mismatched(void); + +#ifdef TOR_UNIT_TESTS +void tor_zstd_set_static_apis_disabled_for_testing(int disabled); +#endif + +#endif /* !defined(TOR_COMPRESS_ZSTD_H) */ + diff --git a/src/common/confline.c b/src/common/confline.c new file mode 100644 index 0000000000..bf613ab742 --- /dev/null +++ b/src/common/confline.c @@ -0,0 +1,538 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "compat.h" +#include "confline.h" +#include "torlog.h" +#include "util.h" +#include "container.h" + +static int config_get_lines_aux(const char *string, config_line_t **result, + int extended, int allow_include, + int *has_include, smartlist_t *opened_lst, + int recursion_level, config_line_t **last); +static smartlist_t *config_get_file_list(const char *path, + smartlist_t *opened_files); +static int config_get_included_config(const char *path, int recursion_level, + int extended, config_line_t **config, + config_line_t **config_last, + smartlist_t *opened_lst); +static int config_process_include(const char *path, int recursion_level, + int extended, config_line_t **list, + config_line_t **list_last, + smartlist_t *opened_lst); + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * append it to *<b>lst</b>. */ +void +config_line_append(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = NULL; + while (*lst) + lst = &((*lst)->next); + + (*lst) = newline; +} + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * and prepend it to *<b>lst</b> */ +void +config_line_prepend(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = *lst; + *lst = newline; +} + +/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or + * NULL if no such key exists. + * + * (In options parsing, this is for handling commandline-only options only; + * other options should be looked up in the appropriate data structure.) */ +const config_line_t * +config_line_find(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcmp(cl->key, key)) + return cl; + } + return NULL; +} + +/** Auxiliary function that does all the work of config_get_lines. + * <b>recursion_level</b> is the count of how many nested %includes we have. + * <b>opened_lst</b> will have a list of opened files if provided. + * Returns the a pointer to the last element of the <b>result</b> in + * <b>last</b>. */ +static int +config_get_lines_aux(const char *string, config_line_t **result, int extended, + int allow_include, int *has_include, + smartlist_t *opened_lst, int recursion_level, + config_line_t **last) +{ + config_line_t *list = NULL, **next, *list_last = NULL; + char *k, *v; + const char *parse_err; + int include_used = 0; + + if (recursion_level > MAX_INCLUDE_RECURSION_LEVEL) { + log_warn(LD_CONFIG, "Error while parsing configuration: more than %d " + "nested %%includes.", MAX_INCLUDE_RECURSION_LEVEL); + return -1; + } + + next = &list; + do { + k = v = NULL; + string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err); + if (!string) { + log_warn(LD_CONFIG, "Error while parsing configuration: %s", + parse_err?parse_err:"<unknown>"); + config_free_lines(list); + tor_free(k); + tor_free(v); + return -1; + } + if (k && v) { + unsigned command = CONFIG_LINE_NORMAL; + if (extended) { + if (k[0] == '+') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + command = CONFIG_LINE_APPEND; + } else if (k[0] == '/') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + tor_free(v); + v = tor_strdup(""); + command = CONFIG_LINE_CLEAR; + } + } + + if (allow_include && !strcmp(k, "%include")) { + tor_free(k); + include_used = 1; + + config_line_t *include_list; + if (config_process_include(v, recursion_level, extended, &include_list, + &list_last, opened_lst) < 0) { + log_warn(LD_CONFIG, "Error reading included configuration " + "file or directory: \"%s\".", v); + config_free_lines(list); + tor_free(v); + return -1; + } + *next = include_list; + if (list_last) + next = &list_last->next; + tor_free(v); + } else { + /* This list can get long, so we keep a pointer to the end of it + * rather than using config_line_append over and over and getting + * n^2 performance. */ + *next = tor_malloc_zero(sizeof(**next)); + (*next)->key = k; + (*next)->value = v; + (*next)->next = NULL; + (*next)->command = command; + list_last = *next; + next = &((*next)->next); + } + } else { + tor_free(k); + tor_free(v); + } + } while (*string); + + if (last) { + *last = list_last; + } + if (has_include) { + *has_include = include_used; + } + *result = list; + return 0; +} + +/** Helper: parse the config string and strdup into key/value + * strings. Set *result to the list, or NULL if parsing the string + * failed. Set *has_include to 1 if <b>result</b> has values from + * %included files. <b>opened_lst</b> will have a list of opened files if + * provided. Return 0 on success, -1 on failure. Warn and ignore any + * misformatted lines. + * + * If <b>extended</b> is set, then treat keys beginning with / and with + as + * indicating "clear" and "append" respectively. */ +int +config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include, + smartlist_t *opened_lst) +{ + return config_get_lines_aux(string, result, extended, 1, has_include, + opened_lst, 1, NULL); +} + +/** Same as config_get_lines_include but does not allow %include */ +int +config_get_lines(const char *string, config_line_t **result, int extended) +{ + return config_get_lines_aux(string, result, extended, 0, NULL, NULL, 1, + NULL); +} + +/** Adds a list of configuration files present on <b>path</b> to + * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file, + * only that file will be added to <b>file_list</b>. If it is a directory, + * all paths for files on that directory root (no recursion) except for files + * whose name starts with a dot will be added to <b>file_list</b>. + * <b>opened_files</b> will have a list of files opened by this function + * if provided. Return 0 on success, -1 on failure. Ignores empty files. + */ +static smartlist_t * +config_get_file_list(const char *path, smartlist_t *opened_files) +{ + smartlist_t *file_list = smartlist_new(); + + if (opened_files) { + smartlist_add_strdup(opened_files, path); + } + + file_status_t file_type = file_status(path); + if (file_type == FN_FILE) { + smartlist_add_strdup(file_list, path); + return file_list; + } else if (file_type == FN_DIR) { + smartlist_t *all_files = tor_listdir(path); + if (!all_files) { + smartlist_free(file_list); + return NULL; + } + smartlist_sort_strings(all_files); + SMARTLIST_FOREACH_BEGIN(all_files, char *, f) { + if (f[0] == '.') { + tor_free(f); + continue; + } + + char *fullname; + tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f); + tor_free(f); + + if (opened_files) { + smartlist_add_strdup(opened_files, fullname); + } + + if (file_status(fullname) != FN_FILE) { + tor_free(fullname); + continue; + } + smartlist_add(file_list, fullname); + } SMARTLIST_FOREACH_END(f); + smartlist_free(all_files); + return file_list; + } else if (file_type == FN_EMPTY) { + return file_list; + } else { + smartlist_free(file_list); + return NULL; + } +} + +/** Creates a list of config lines present on included <b>path</b>. + * Set <b>config</b> to the list and <b>config_last</b> to the last element of + * <b>config</b>. <b>opened_lst</b> will have a list of opened files if + * provided. Return 0 on success, -1 on failure. */ +static int +config_get_included_config(const char *path, int recursion_level, int extended, + config_line_t **config, config_line_t **config_last, + smartlist_t *opened_lst) +{ + char *included_conf = read_file_to_str(path, 0, NULL); + if (!included_conf) { + return -1; + } + + if (config_get_lines_aux(included_conf, config, extended, 1, NULL, + opened_lst, recursion_level+1, config_last) < 0) { + tor_free(included_conf); + return -1; + } + + tor_free(included_conf); + return 0; +} + +/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the + * list of configuration settings obtained and <b>list_last</b> to the last + * element of the same list. <b>opened_lst</b> will have a list of opened + * files if provided. Return 0 on success, -1 on failure. */ +static int +config_process_include(const char *path, int recursion_level, int extended, + config_line_t **list, config_line_t **list_last, + smartlist_t *opened_lst) +{ + config_line_t *ret_list = NULL; + config_line_t **next = &ret_list; + + smartlist_t *config_files = config_get_file_list(path, opened_lst); + if (!config_files) { + return -1; + } + + int rv = -1; + SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) { + config_line_t *included_config = NULL; + if (config_get_included_config(config_file, recursion_level, extended, + &included_config, list_last, + opened_lst) < 0) { + goto done; + } + + *next = included_config; + if (*list_last) + next = &(*list_last)->next; + + } SMARTLIST_FOREACH_END(config_file); + *list = ret_list; + rv = 0; + + done: + SMARTLIST_FOREACH(config_files, char *, f, tor_free(f)); + smartlist_free(config_files); + return rv; +} + +/** + * Free all the configuration lines on the linked list <b>front</b>. + */ +void +config_free_lines_(config_line_t *front) +{ + config_line_t *tmp; + + while (front) { + tmp = front; + front = tmp->next; + + tor_free(tmp->key); + tor_free(tmp->value); + tor_free(tmp); + } +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>. */ +config_line_t * +config_lines_dup(const config_line_t *inp) +{ + return config_lines_dup_and_filter(inp, NULL); +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>, + * but only the ones whose keys begin with <b>key</b> (case-insensitive). + * If <b>key</b> is NULL, do not filter. */ +config_line_t * +config_lines_dup_and_filter(const config_line_t *inp, + const char *key) +{ + config_line_t *result = NULL; + config_line_t **next_out = &result; + while (inp) { + if (key && strcasecmpstart(inp->key, key)) { + inp = inp->next; + continue; + } + *next_out = tor_malloc_zero(sizeof(config_line_t)); + (*next_out)->key = tor_strdup(inp->key); + (*next_out)->value = tor_strdup(inp->value); + inp = inp->next; + next_out = &((*next_out)->next); + } + (*next_out) = NULL; + return result; +} + +/** Return true iff a and b contain identical keys and values in identical + * order. */ +int +config_lines_eq(config_line_t *a, config_line_t *b) +{ + while (a && b) { + if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) + return 0; + a = a->next; + b = b->next; + } + if (a || b) + return 0; + return 1; +} + +/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */ +int +config_count_key(const config_line_t *a, const char *key) +{ + int n = 0; + while (a) { + if (!strcasecmp(a->key, key)) { + ++n; + } + a = a->next; + } + return n; +} + +/** Given a string containing part of a configuration file or similar format, + * advance past comments and whitespace and try to parse a single line. If we + * parse a line successfully, set *<b>key_out</b> to a new string holding the + * key portion and *<b>value_out</b> to a new string holding the value portion + * of the line, and return a pointer to the start of the next line. If we run + * out of data, return a pointer to the end of the string. If we encounter an + * error, return NULL and set *<b>err_out</b> (if provided) to an error + * message. + */ +const char * +parse_config_line_from_str_verbose(const char *line, char **key_out, + char **value_out, + const char **err_out) +{ + /* + See torrc_format.txt for a description of the (silly) format this parses. + */ + const char *key, *val, *cp; + int continuation = 0; + + tor_assert(key_out); + tor_assert(value_out); + + *key_out = *value_out = NULL; + key = val = NULL; + /* Skip until the first keyword. */ + while (1) { + while (TOR_ISSPACE(*line)) + ++line; + if (*line == '#') { + while (*line && *line != '\n') + ++line; + } else { + break; + } + } + + if (!*line) { /* End of string? */ + *key_out = *value_out = NULL; + return line; + } + + /* Skip until the next space or \ followed by newline. */ + key = line; + while (*line && !TOR_ISSPACE(*line) && *line != '#' && + ! (line[0] == '\\' && line[1] == '\n')) + ++line; + *key_out = tor_strndup(key, line-key); + + /* Skip until the value. */ + while (*line == ' ' || *line == '\t') + ++line; + + val = line; + + /* Find the end of the line. */ + if (*line == '\"') { // XXX No continuation handling is done here + if (!(line = unescape_string(line, value_out, NULL))) { + if (err_out) + *err_out = "Invalid escape sequence in quoted string"; + return NULL; + } + while (*line == ' ' || *line == '\t') + ++line; + if (*line == '\r' && *(++line) == '\n') + ++line; + if (*line && *line != '#' && *line != '\n') { + if (err_out) + *err_out = "Excess data after quoted string"; + return NULL; + } + } else { + /* Look for the end of the line. */ + while (*line && *line != '\n' && (*line != '#' || continuation)) { + if (*line == '\\' && line[1] == '\n') { + continuation = 1; + line += 2; + } else if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + if (*line == '\n') + ++line; + } else { + ++line; + } + } + + if (*line == '\n') { + cp = line++; + } else { + cp = line; + } + /* Now back cp up to be the last nonspace character */ + while (cp>val && TOR_ISSPACE(*(cp-1))) + --cp; + + tor_assert(cp >= val); + + /* Now copy out and decode the value. */ + *value_out = tor_strndup(val, cp-val); + if (continuation) { + char *v_out, *v_in; + v_out = v_in = *value_out; + while (*v_in) { + if (*v_in == '#') { + do { + ++v_in; + } while (*v_in && *v_in != '\n'); + if (*v_in == '\n') + ++v_in; + } else if (v_in[0] == '\\' && v_in[1] == '\n') { + v_in += 2; + } else { + *v_out++ = *v_in++; + } + } + *v_out = '\0'; + } + } + + if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + } + while (TOR_ISSPACE(*line)) ++line; + + return line; +} + diff --git a/src/common/confline.h b/src/common/confline.h new file mode 100644 index 0000000000..772a9bbbdc --- /dev/null +++ b/src/common/confline.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONFLINE_H +#define TOR_CONFLINE_H + +#include "container.h" + +/** Ordinary configuration line. */ +#define CONFIG_LINE_NORMAL 0 +/** Appends to previous configuration for the same option, even if we + * would ordinary replace it. */ +#define CONFIG_LINE_APPEND 1 +/* Removes all previous configuration for an option. */ +#define CONFIG_LINE_CLEAR 2 + +#define MAX_INCLUDE_RECURSION_LEVEL 31 + +/** A linked list of lines in a config file, or elsewhere */ +typedef struct config_line_t { + char *key; + char *value; + struct config_line_t *next; + + /** What special treatment (if any) does this line require? */ + unsigned int command:2; + /** If true, subsequent assignments to this linelist should replace + * it, not extend it. Set only on the first item in a linelist in an + * or_options_t. */ + unsigned int fragile:1; +} config_line_t; + +void config_line_append(config_line_t **lst, + const char *key, const char *val); +void config_line_prepend(config_line_t **lst, + const char *key, const char *val); +config_line_t *config_lines_dup(const config_line_t *inp); +config_line_t *config_lines_dup_and_filter(const config_line_t *inp, + const char *key); +const config_line_t *config_line_find(const config_line_t *lines, + const char *key); +int config_lines_eq(config_line_t *a, config_line_t *b); +int config_count_key(const config_line_t *a, const char *key); +int config_get_lines(const char *string, config_line_t **result, int extended); +int config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include, + smartlist_t *opened_lst); +void config_free_lines_(config_line_t *front); +#define config_free_lines(front) \ + do { \ + config_free_lines_(front); \ + (front) = NULL; \ + } while (0) +const char *parse_config_line_from_str_verbose(const char *line, + char **key_out, char **value_out, + const char **err_out); +#endif /* !defined(TOR_CONFLINE_H) */ + diff --git a/src/common/container.c b/src/common/container.c index ec59dccf62..72ad3a9258 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,7 +15,7 @@ #include "util.h" #include "torlog.h" #include "container.h" -#include "crypto.h" +#include "crypto_digest.h" #include <stdlib.h> #include <string.h> @@ -42,7 +42,7 @@ smartlist_new,(void)) * list's elements. */ MOCK_IMPL(void, -smartlist_free,(smartlist_t *sl)) +smartlist_free_,(smartlist_t *sl)) { if (!sl) return; @@ -132,6 +132,24 @@ smartlist_remove(smartlist_t *sl, const void *element) } } +/** As <b>smartlist_remove</b>, but do not change the order of + * any elements not removed */ +void +smartlist_remove_keeporder(smartlist_t *sl, const void *element) +{ + int i, j, num_used_orig = sl->num_used; + if (element == NULL) + return; + + for (i=j=0; j < num_used_orig; ++j) { + if (sl->list[j] == element) { + --sl->num_used; + } else { + sl->list[i++] = sl->list[j]; + } + } +} + /** If <b>sl</b> is nonempty, remove and return the final element. Otherwise, * return NULL. */ void * @@ -610,7 +628,7 @@ smartlist_uniq(smartlist_t *sl, * less than member, and greater than 0 if key is greater then member. */ void * -smartlist_bsearch(smartlist_t *sl, const void *key, +smartlist_bsearch(const smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) { int found, idx; @@ -825,13 +843,13 @@ smartlist_sort_pointers(smartlist_t *sl) * } * * void timer_heap_insert(smartlist_t *heap, timer_t *timer) { - * smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index), + * smartlist_pqueue_add(heap, compare, offsetof(timer_t, heap_index), * timer); * } * * void timer_heap_pop(smartlist_t *heap) { * return smartlist_pqueue_pop(heap, compare, - * STRUCT_OFFSET(timer_t, heap_index)); + * offsetof(timer_t, heap_index)); * } */ @@ -1145,19 +1163,26 @@ HT_GENERATE2(digest256map_impl, digest256map_entry_t, node, digest256map_entry_hash, digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_) +#define strmap_entry_free(ent) \ + FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent)) +#define digestmap_entry_free(ent) \ + FREE_AND_NULL(digestmap_entry_t, digestmap_entry_free_, (ent)) +#define digest256map_entry_free(ent) \ + FREE_AND_NULL(digest256map_entry_t, digest256map_entry_free_, (ent)) + static inline void -strmap_entry_free(strmap_entry_t *ent) +strmap_entry_free_(strmap_entry_t *ent) { tor_free(ent->key); tor_free(ent); } static inline void -digestmap_entry_free(digestmap_entry_t *ent) +digestmap_entry_free_(digestmap_entry_t *ent) { tor_free(ent); } static inline void -digest256map_entry_free(digest256map_entry_t *ent) +digest256map_entry_free_(digest256map_entry_t *ent) { tor_free(ent); } @@ -1317,7 +1342,7 @@ digest256map_assign_key(digest256map_entry_t *ent, const uint8_t *key) * those entries. If free_val is provided, invoked it every value in \ * <b>map</b>. */ \ MOCK_IMPL(void, \ - prefix##_free, (maptype *map, void (*free_val)(void*))) \ + prefix##_free_, (maptype *map, void (*free_val)(void*))) \ { \ prefix##_entry_t **ent, **next, *this; \ if (!map) \ @@ -1507,7 +1532,7 @@ digestset_new(int max_elements) /** Free all storage held in <b>set</b>. */ void -digestset_free(digestset_t *set) +digestset_free_(digestset_t *set) { if (!set) return; diff --git a/src/common/container.h b/src/common/container.h index 71495b660a..7457c1e918 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONTAINER_H @@ -28,11 +28,14 @@ typedef struct smartlist_t { } smartlist_t; MOCK_DECL(smartlist_t *, smartlist_new, (void)); -MOCK_DECL(void, smartlist_free, (smartlist_t *sl)); +MOCK_DECL(void, smartlist_free_, (smartlist_t *sl)); +#define smartlist_free(sl) FREE_AND_NULL(smartlist_t, smartlist_free_, (sl)) + void smartlist_clear(smartlist_t *sl); void smartlist_add(smartlist_t *sl, void *element); void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2); void smartlist_remove(smartlist_t *sl, const void *element); +void smartlist_remove_keeporder(smartlist_t *sl, const void *element); void *smartlist_pop_last(smartlist_t *sl); void smartlist_reverse(smartlist_t *sl); void smartlist_string_remove(smartlist_t *sl, const char *element); @@ -73,11 +76,11 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) { tor_assert(sl->num_used > idx); sl->list[idx] = val; } -#else +#else /* !(defined(DEBUG_SMARTLIST)) */ #define smartlist_len(sl) ((sl)->num_used) #define smartlist_get(sl, idx) ((sl)->list[idx]) #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val)) -#endif +#endif /* defined(DEBUG_SMARTLIST) */ /** Exchange the elements at indices <b>idx1</b> and <b>idx2</b> of the * smartlist <b>sl</b>. */ @@ -117,7 +120,7 @@ const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl); void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); void smartlist_uniq_digests256(smartlist_t *sl); -void *smartlist_bsearch(smartlist_t *sl, const void *key, +void *smartlist_bsearch(const smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)); int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member), @@ -223,6 +226,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ + (void) var ## _sl_idx; \ } STMT_END /** @@ -348,7 +352,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, void* prefix##set(maptype *map, keytype key, void *val); \ void* prefix##get(const maptype *map, keytype key); \ void* prefix##remove(maptype *map, keytype key); \ - MOCK_DECL(void, prefix##free, (maptype *map, void (*free_val)(void*))); \ + MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \ int prefix##isempty(const maptype *map); \ int prefix##size(const maptype *map); \ prefix##iter_t *prefix##iter_init(maptype *map); \ @@ -366,6 +370,16 @@ DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_); * table. */ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); +#define MAP_FREE_AND_NULL(maptype, map, fn) \ + do { \ + maptype ## _free_((map), (fn)); \ + (map) = NULL; \ + } while (0) + +#define strmap_free(map, fn) MAP_FREE_AND_NULL(strmap, (map), (fn)) +#define digestmap_free(map, fn) MAP_FREE_AND_NULL(digestmap, (map), (fn)) +#define digest256map_free(map, fn) MAP_FREE_AND_NULL(digest256map, (map), (fn)) + #undef DECLARE_MAP_FNS /** Iterates over the key-value pairs in a map <b>map</b> in order. @@ -526,9 +540,9 @@ void* strmap_remove_lc(strmap_t *map, const char *key); return (valtype*)digestmap_remove((digestmap_t*)map, key); \ } \ ATTR_UNUSED static inline void \ - prefix##f##ree(maptype *map, void (*free_val)(void*)) \ + prefix##f##ree_(maptype *map, void (*free_val)(void*)) \ { \ - digestmap_free((digestmap_t*)map, free_val); \ + digestmap_free_((digestmap_t*)map, free_val); \ } \ ATTR_UNUSED static inline int \ prefix##isempty(maptype *map) \ @@ -578,7 +592,7 @@ void* strmap_remove_lc(strmap_t *map, const char *key); #define BITARRAY_SHIFT 6 #else #error "int is neither 4 nor 8 bytes. I can't deal with that." -#endif +#endif /* SIZEOF_INT == 4 || ... */ #define BITARRAY_MASK ((1u<<BITARRAY_SHIFT)-1) /** A random-access array of one-bit-wide elements. */ @@ -612,10 +626,12 @@ bitarray_expand(bitarray_t *ba, } /** Free the bit array <b>ba</b>. */ static inline void -bitarray_free(bitarray_t *ba) +bitarray_free_(bitarray_t *ba) { tor_free(ba); } +#define bitarray_free(ba) FREE_AND_NULL(bitarray_t, bitarray_free_, (ba)) + /** Set the <b>bit</b>th bit in <b>b</b> to 1. */ static inline void bitarray_set(bitarray_t *b, int bit) @@ -677,7 +693,8 @@ digestset_contains(const digestset_t *set, const char *digest) #undef BIT digestset_t *digestset_new(int max_elements); -void digestset_free(digestset_t* set); +void digestset_free_(digestset_t* set); +#define digestset_free(set) FREE_AND_NULL(digestset_t, digestset_free_, (set)) /* These functions, given an <b>array</b> of <b>n_elements</b>, return the * <b>nth</b> lowest element. <b>nth</b>=0 gives the lowest element; @@ -721,5 +738,5 @@ third_quartile_uint32(uint32_t *array, int n_elements) return find_nth_uint32(array, n_elements, (n_elements*3)/4); } -#endif +#endif /* !defined(TOR_CONTAINER_H) */ diff --git a/src/common/crypto.c b/src/common/crypto.c index f8495bb107..d5b7c96916 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,27 +20,29 @@ /* Windows defines this; so does OpenSSL 0.9.8h and later. We don't actually * use either definition. */ #undef OCSP_RESPONSE -#endif +#endif /* defined(_WIN32) */ #define CRYPTO_PRIVATE -#include "crypto.h" #include "compat_openssl.h" +#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_rand.h" +#include "crypto_rsa.h" +#include "crypto_util.h" DISABLE_GCC_WARNING(redundant-decls) #include <openssl/err.h> -#include <openssl/rsa.h> -#include <openssl/pem.h> #include <openssl/evp.h> #include <openssl/engine.h> -#include <openssl/rand.h> #include <openssl/bn.h> #include <openssl/dh.h> #include <openssl/conf.h> #include <openssl/hmac.h> +#include <openssl/ssl.h> ENABLE_GCC_WARNING(redundant-decls) @@ -50,7 +52,7 @@ ENABLE_GCC_WARNING(redundant-decls) #else #pragma GCC diagnostic warning "-Wredundant-decls" #endif -#endif +#endif /* __GNUC__ && GCC_VERSION >= 402 */ #ifdef HAVE_CTYPE_H #include <ctype.h> @@ -58,18 +60,6 @@ ENABLE_GCC_WARNING(redundant-decls) #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_FCNTL_H -#include <sys/fcntl.h> -#endif -#ifdef HAVE_SYS_SYSCALL_H -#include <sys/syscall.h> -#endif -#ifdef HAVE_SYS_RANDOM_H -#include <sys/random.h> -#endif #include "torlog.h" #include "torint.h" @@ -82,80 +72,14 @@ ENABLE_GCC_WARNING(redundant-decls) #include "keccak-tiny/keccak-tiny.h" -#ifdef ANDROID -/* Android's OpenSSL seems to have removed all of its Engine support. */ -#define DISABLE_ENGINES -#endif - -#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && \ - !defined(LIBRESSL_VERSION_NUMBER) -/* OpenSSL as of 1.1.0pre4 has an "new" thread API, which doesn't require - * seting up various callbacks. - * - * OpenSSL 1.1.0pre4 has a messed up `ERR_remove_thread_state()` prototype, - * while the previous one was restored in pre5, and the function made a no-op - * (along with a deprecated annotation, which produces a compiler warning). - * - * While it is possible to support all three versions of the thread API, - * a version that existed only for one snapshot pre-release is kind of - * pointless, so let's not. - */ -#define NEW_THREAD_API -#endif - -/** Longest recognized */ -#define MAX_DNS_LABEL_SIZE 63 - -/** Largest strong entropy request */ -#define MAX_STRONGEST_RAND_SIZE 256 - -#ifndef NEW_THREAD_API -/** A number of preallocated mutexes for use by OpenSSL. */ -static tor_mutex_t **openssl_mutexes_ = NULL; -/** How many mutexes have we allocated for use by OpenSSL? */ -static int n_openssl_mutexes_ = 0; -#endif - -/** A public key, or a public/private key-pair. */ -struct crypto_pk_t -{ - int refs; /**< reference count, so we don't have to copy keys */ - RSA *key; /**< The key itself */ -}; - /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake * while we're waiting for the second.*/ struct crypto_dh_t { DH *dh; /**< The openssl DH object */ }; -static int setup_openssl_threading(void); static int tor_check_dh_key(int severity, const BIGNUM *bn); -/** Return the number of bytes added by padding method <b>padding</b>. - */ -static inline int -crypto_get_rsa_padding_overhead(int padding) -{ - switch (padding) - { - case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD; - default: tor_assert(0); return -1; // LCOV_EXCL_LINE - } -} - -/** Given a padding method <b>padding</b>, return the correct OpenSSL constant. - */ -static inline int -crypto_get_rsa_padding(int padding) -{ - switch (padding) - { - case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING; - default: tor_assert(0); return -1; // LCOV_EXCL_LINE - } -} - /** Boolean: has OpenSSL's crypto been initialized? */ static int crypto_early_initialized_ = 0; @@ -198,7 +122,7 @@ log_engine(const char *fn, ENGINE *e) log_info(LD_CRYPTO, "Using default implementation for %s", fn); } } -#endif +#endif /* !defined(DISABLE_ENGINES) */ #ifndef DISABLE_ENGINES /** Try to load an engine in a shared library via fully qualified path. @@ -218,76 +142,14 @@ try_load_engine(const char *path, const char *engine) } return e; } -#endif - -/* Returns a trimmed and human-readable version of an openssl version string -* <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10 -* May 2012' and this will parse them into a form similar to '1.0.0b' */ -static char * -parse_openssl_version_str(const char *raw_version) -{ - const char *end_of_version = NULL; - /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's - trim that down. */ - if (!strcmpstart(raw_version, "OpenSSL ")) { - raw_version += strlen("OpenSSL "); - end_of_version = strchr(raw_version, ' '); - } - - if (end_of_version) - return tor_strndup(raw_version, - end_of_version-raw_version); - else - return tor_strdup(raw_version); -} - -static char *crypto_openssl_version_str = NULL; -/* Return a human-readable version of the run-time openssl version number. */ -const char * -crypto_openssl_get_version_str(void) -{ - if (crypto_openssl_version_str == NULL) { - const char *raw_version = OpenSSL_version(OPENSSL_VERSION); - crypto_openssl_version_str = parse_openssl_version_str(raw_version); - } - return crypto_openssl_version_str; -} - -static char *crypto_openssl_header_version_str = NULL; -/* Return a human-readable version of the compile-time openssl version -* number. */ -const char * -crypto_openssl_get_header_version_str(void) -{ - if (crypto_openssl_header_version_str == NULL) { - crypto_openssl_header_version_str = - parse_openssl_version_str(OPENSSL_VERSION_TEXT); - } - return crypto_openssl_header_version_str; -} +#endif /* !defined(DISABLE_ENGINES) */ -/** Make sure that openssl is using its default PRNG. Return 1 if we had to - * adjust it; 0 otherwise. */ -STATIC int -crypto_force_rand_ssleay(void) -{ - RAND_METHOD *default_method; - default_method = RAND_OpenSSL(); - if (RAND_get_rand_method() != default_method) { - log_notice(LD_CRYPTO, "It appears that one of our engines has provided " - "a replacement the OpenSSL RNG. Resetting it to the default " - "implementation."); - RAND_set_rand_method(default_method); - return 1; - } - return 0; -} +static int have_seeded_siphash = 0; /** Set up the siphash key if we haven't already done so. */ int crypto_init_siphash_key(void) { - static int have_seeded_siphash = 0; struct sipkey key; if (have_seeded_siphash) return 0; @@ -307,8 +169,15 @@ crypto_early_init(void) crypto_early_initialized_ = 1; +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS | + OPENSSL_INIT_LOAD_CRYPTO_STRINGS | + OPENSSL_INIT_ADD_ALL_CIPHERS | + OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); +#else ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); +#endif setup_openssl_threading(); @@ -394,7 +263,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) #else log_engine("ECDH", ENGINE_get_default_ECDH()); log_engine("ECDSA", ENGINE_get_default_ECDSA()); -#endif +#endif /* defined(OPENSSL_1_1_API) */ log_engine("RAND", ENGINE_get_default_RAND()); log_engine("RAND (which we will not use)", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); @@ -412,7 +281,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm)); #endif -#endif +#endif /* defined(DISABLE_ENGINES) */ } else { log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } @@ -437,73 +306,6 @@ crypto_thread_cleanup(void) #endif } -/** used internally: quicly validate a crypto_pk_t object as a private key. - * Return 1 iff the public key is valid, 0 if obviously invalid. - */ -static int -crypto_pk_private_ok(const crypto_pk_t *k) -{ -#ifdef OPENSSL_1_1_API - if (!k || !k->key) - return 0; - - const BIGNUM *p, *q; - RSA_get0_factors(k->key, &p, &q); - return p != NULL; /* XXX/yawning: Should we check q? */ -#else - return k && k->key && k->key->p; -#endif -} - -/** used by tortls.c: wrap an RSA* in a crypto_pk_t. */ -crypto_pk_t * -crypto_new_pk_from_rsa_(RSA *rsa) -{ - crypto_pk_t *env; - tor_assert(rsa); - env = tor_malloc(sizeof(crypto_pk_t)); - env->refs = 1; - env->key = rsa; - return env; -} - -/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a - * crypto_pk_t. */ -RSA * -crypto_pk_get_rsa_(crypto_pk_t *env) -{ - return env->key; -} - -/** used by tortls.c: get an equivalent EVP_PKEY* for a crypto_pk_t. Iff - * private is set, include the private-key portion of the key. Return a valid - * pointer on success, and NULL on failure. */ -MOCK_IMPL(EVP_PKEY *, - crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) -{ - RSA *key = NULL; - EVP_PKEY *pkey = NULL; - tor_assert(env->key); - if (private) { - if (!(key = RSAPrivateKey_dup(env->key))) - goto error; - } else { - if (!(key = RSAPublicKey_dup(env->key))) - goto error; - } - if (!(pkey = EVP_PKEY_new())) - goto error; - if (!(EVP_PKEY_assign_RSA(pkey, key))) - goto error; - return pkey; - error: - if (pkey) - EVP_PKEY_free(pkey); - if (key) - RSA_free(key); - return NULL; -} - /** Used by tortls.c: Get the DH* from a crypto_dh_t. */ DH * @@ -512,38 +314,6 @@ crypto_dh_get_dh_(crypto_dh_t *dh) return dh->dh; } -/** Allocate and return storage for a public key. The key itself will not yet - * be set. - */ -MOCK_IMPL(crypto_pk_t *, - crypto_pk_new,(void)) -{ - RSA *rsa; - - rsa = RSA_new(); - tor_assert(rsa); - return crypto_new_pk_from_rsa_(rsa); -} - -/** Release a reference to an asymmetric key; when all the references - * are released, free the key. - */ -void -crypto_pk_free(crypto_pk_t *env) -{ - if (!env) - return; - - if (--env->refs > 0) - return; - tor_assert(env->refs == 0); - - if (env->key) - RSA_free(env->key); - - tor_free(env); -} - /** Allocate and return a new symmetric cipher using the provided key and iv. * The key is <b>bits</b> bits long; the IV is CIPHER_IV_LEN bytes. Both * must be provided. Key length must be 128, 192, or 256 */ @@ -592,7 +362,7 @@ crypto_cipher_new(const char *key) /** Free a symmetric cipher. */ void -crypto_cipher_free(crypto_cipher_t *env) +crypto_cipher_free_(crypto_cipher_t *env) { if (!env) return; @@ -600,865 +370,6 @@ crypto_cipher_free(crypto_cipher_t *env) aes_cipher_free(env); } -/* public key crypto */ - -/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>. - * Return 0 on success, -1 on failure. - */ -MOCK_IMPL(int, - crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) -{ - tor_assert(env); - - if (env->key) { - RSA_free(env->key); - env->key = NULL; - } - - { - BIGNUM *e = BN_new(); - RSA *r = NULL; - if (!e) - goto done; - if (! BN_set_word(e, 65537)) - goto done; - r = RSA_new(); - if (!r) - goto done; - if (RSA_generate_key_ex(r, bits, e, NULL) == -1) - goto done; - - env->key = r; - r = NULL; - done: - if (e) - BN_clear_free(e); - if (r) - RSA_free(r); - } - - if (!env->key) { - crypto_log_errors(LOG_WARN, "generating RSA key"); - return -1; - } - - return 0; -} - -/** A PEM callback that always reports a failure to get a password */ -static int -pem_no_password_cb(char *buf, int size, int rwflag, void *u) -{ - (void)buf; - (void)size; - (void)rwflag; - (void)u; - /* The openssl documentation says that a callback "must" return 0 if an - * error occurred. But during the 1.1.1 series (commit c82c3462267afdbbaa5 - * they changed the interpretation so that 0 indicates an empty password and - * -1 indicates an error. We want to reject any encrypted PEM buffers, so we - * return -1. This will work on older OpenSSL versions and LibreSSL too. */ - return -1; -} - -/** Read a PEM-encoded private key from the <b>len</b>-byte string <b>s</b> - * into <b>env</b>. Return 0 on success, -1 on failure. If len is -1, - * the string is nul-terminated. - */ -int -crypto_pk_read_private_key_from_string(crypto_pk_t *env, - const char *s, ssize_t len) -{ - BIO *b; - - tor_assert(env); - tor_assert(s); - tor_assert(len < INT_MAX && len < SSIZE_T_CEILING); - - /* Create a read-only memory BIO, backed by the string 's' */ - b = BIO_new_mem_buf((char*)s, (int)len); - if (!b) - return -1; - - if (env->key) - RSA_free(env->key); - - env->key = PEM_read_bio_RSAPrivateKey(b,NULL,pem_no_password_cb,NULL); - - BIO_free(b); - - if (!env->key) { - crypto_log_errors(LOG_WARN, "Error parsing private key"); - return -1; - } - return 0; -} - -/** Read a PEM-encoded private key from the file named by - * <b>keyfile</b> into <b>env</b>. Return 0 on success, -1 on failure. - */ -int -crypto_pk_read_private_key_from_filename(crypto_pk_t *env, - const char *keyfile) -{ - char *contents; - int r; - - /* Read the file into a string. */ - contents = read_file_to_str(keyfile, 0, NULL); - if (!contents) { - log_warn(LD_CRYPTO, "Error reading private key from \"%s\"", keyfile); - return -1; - } - - /* Try to parse it. */ - r = crypto_pk_read_private_key_from_string(env, contents, -1); - memwipe(contents, 0, strlen(contents)); - tor_free(contents); - if (r) - return -1; /* read_private_key_from_string already warned, so we don't.*/ - - /* Make sure it's valid. */ - if (crypto_pk_check_key(env) <= 0) - return -1; - - return 0; -} - -/** Helper function to implement crypto_pk_write_*_key_to_string. Return 0 on - * success, -1 on failure. */ -static int -crypto_pk_write_key_to_string_impl(crypto_pk_t *env, char **dest, - size_t *len, int is_public) -{ - BUF_MEM *buf; - BIO *b; - int r; - - tor_assert(env); - tor_assert(env->key); - tor_assert(dest); - - b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ - if (!b) - return -1; - - /* Now you can treat b as if it were a file. Just use the - * PEM_*_bio_* functions instead of the non-bio variants. - */ - if (is_public) - r = PEM_write_bio_RSAPublicKey(b, env->key); - else - r = PEM_write_bio_RSAPrivateKey(b, env->key, NULL,NULL,0,NULL,NULL); - - if (!r) { - crypto_log_errors(LOG_WARN, "writing RSA key to string"); - BIO_free(b); - return -1; - } - - BIO_get_mem_ptr(b, &buf); - - *dest = tor_malloc(buf->length+1); - memcpy(*dest, buf->data, buf->length); - (*dest)[buf->length] = 0; /* nul terminate it */ - *len = buf->length; - - BIO_free(b); - - return 0; -} - -/** PEM-encode the public key portion of <b>env</b> and write it to a - * newly allocated string. On success, set *<b>dest</b> to the new - * string, *<b>len</b> to the string's length, and return 0. On - * failure, return -1. - */ -int -crypto_pk_write_public_key_to_string(crypto_pk_t *env, char **dest, - size_t *len) -{ - return crypto_pk_write_key_to_string_impl(env, dest, len, 1); -} - -/** PEM-encode the private key portion of <b>env</b> and write it to a - * newly allocated string. On success, set *<b>dest</b> to the new - * string, *<b>len</b> to the string's length, and return 0. On - * failure, return -1. - */ -int -crypto_pk_write_private_key_to_string(crypto_pk_t *env, char **dest, - size_t *len) -{ - return crypto_pk_write_key_to_string_impl(env, dest, len, 0); -} - -/** Read a PEM-encoded public key from the first <b>len</b> characters of - * <b>src</b>, and store the result in <b>env</b>. Return 0 on success, -1 on - * failure. - */ -int -crypto_pk_read_public_key_from_string(crypto_pk_t *env, const char *src, - size_t len) -{ - BIO *b; - - tor_assert(env); - tor_assert(src); - tor_assert(len<INT_MAX); - - b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ - if (!b) - return -1; - - BIO_write(b, src, (int)len); - - if (env->key) - RSA_free(env->key); - env->key = PEM_read_bio_RSAPublicKey(b, NULL, pem_no_password_cb, NULL); - BIO_free(b); - if (!env->key) { - crypto_log_errors(LOG_WARN, "reading public key from string"); - return -1; - } - - return 0; -} - -/** Write the private key from <b>env</b> into the file named by <b>fname</b>, - * PEM-encoded. Return 0 on success, -1 on failure. - */ -int -crypto_pk_write_private_key_to_filename(crypto_pk_t *env, - const char *fname) -{ - BIO *bio; - char *cp; - long len; - char *s; - int r; - - tor_assert(crypto_pk_private_ok(env)); - - if (!(bio = BIO_new(BIO_s_mem()))) - return -1; - if (PEM_write_bio_RSAPrivateKey(bio, env->key, NULL,NULL,0,NULL,NULL) - == 0) { - crypto_log_errors(LOG_WARN, "writing private key"); - BIO_free(bio); - return -1; - } - len = BIO_get_mem_data(bio, &cp); - tor_assert(len >= 0); - s = tor_malloc(len+1); - memcpy(s, cp, len); - s[len]='\0'; - r = write_str_to_file(fname, s, 0); - BIO_free(bio); - memwipe(s, 0, strlen(s)); - tor_free(s); - return r; -} - -/** Return true iff <b>env</b> has a valid key. - */ -int -crypto_pk_check_key(crypto_pk_t *env) -{ - int r; - tor_assert(env); - - r = RSA_check_key(env->key); - if (r <= 0) - crypto_log_errors(LOG_WARN,"checking RSA key"); - return r; -} - -/** Return true iff <b>key</b> contains the private-key portion of the RSA - * key. */ -int -crypto_pk_key_is_private(const crypto_pk_t *key) -{ - tor_assert(key); - return crypto_pk_private_ok(key); -} - -/** Return true iff <b>env</b> contains a public key whose public exponent - * equals 65537. - */ -int -crypto_pk_public_exponent_ok(crypto_pk_t *env) -{ - tor_assert(env); - tor_assert(env->key); - - const BIGNUM *e; - -#ifdef OPENSSL_1_1_API - const BIGNUM *n, *d; - RSA_get0_key(env->key, &n, &e, &d); -#else - e = env->key->e; -#endif - return BN_is_word(e, 65537); -} - -/** Compare the public-key components of a and b. Return less than 0 - * if a\<b, 0 if a==b, and greater than 0 if a\>b. A NULL key is - * considered to be less than all non-NULL keys, and equal to itself. - * - * Note that this may leak information about the keys through timing. - */ -int -crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b) -{ - int result; - char a_is_non_null = (a != NULL) && (a->key != NULL); - char b_is_non_null = (b != NULL) && (b->key != NULL); - char an_argument_is_null = !a_is_non_null | !b_is_non_null; - - result = tor_memcmp(&a_is_non_null, &b_is_non_null, sizeof(a_is_non_null)); - if (an_argument_is_null) - return result; - - const BIGNUM *a_n, *a_e; - const BIGNUM *b_n, *b_e; - -#ifdef OPENSSL_1_1_API - const BIGNUM *a_d, *b_d; - RSA_get0_key(a->key, &a_n, &a_e, &a_d); - RSA_get0_key(b->key, &b_n, &b_e, &b_d); -#else - a_n = a->key->n; - a_e = a->key->e; - b_n = b->key->n; - b_e = b->key->e; -#endif - - tor_assert(a_n != NULL && a_e != NULL); - tor_assert(b_n != NULL && b_e != NULL); - - result = BN_cmp(a_n, b_n); - if (result) - return result; - return BN_cmp(a_e, b_e); -} - -/** Compare the public-key components of a and b. Return non-zero iff - * a==b. A NULL key is considered to be distinct from all non-NULL - * keys, and equal to itself. - * - * Note that this may leak information about the keys through timing. - */ -int -crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b) -{ - return (crypto_pk_cmp_keys(a, b) == 0); -} - -/** Return the size of the public key modulus in <b>env</b>, in bytes. */ -size_t -crypto_pk_keysize(const crypto_pk_t *env) -{ - tor_assert(env); - tor_assert(env->key); - - return (size_t) RSA_size((RSA*)env->key); -} - -/** Return the size of the public key modulus of <b>env</b>, in bits. */ -int -crypto_pk_num_bits(crypto_pk_t *env) -{ - tor_assert(env); - tor_assert(env->key); - -#ifdef OPENSSL_1_1_API - /* It's so stupid that there's no other way to check that n is valid - * before calling RSA_bits(). - */ - const BIGNUM *n, *e, *d; - RSA_get0_key(env->key, &n, &e, &d); - tor_assert(n != NULL); - - return RSA_bits(env->key); -#else - tor_assert(env->key->n); - return BN_num_bits(env->key->n); -#endif -} - -/** Increase the reference count of <b>env</b>, and return it. - */ -crypto_pk_t * -crypto_pk_dup_key(crypto_pk_t *env) -{ - tor_assert(env); - tor_assert(env->key); - - env->refs++; - return env; -} - -#ifdef TOR_UNIT_TESTS -/** For testing: replace dest with src. (Dest must have a refcount - * of 1) */ -void -crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src) -{ - tor_assert(dest); - tor_assert(dest->refs == 1); - tor_assert(src); - RSA_free(dest->key); - dest->key = RSAPrivateKey_dup(src->key); -} -#endif - -/** Make a real honest-to-goodness copy of <b>env</b>, and return it. - * Returns NULL on failure. */ -crypto_pk_t * -crypto_pk_copy_full(crypto_pk_t *env) -{ - RSA *new_key; - int privatekey = 0; - tor_assert(env); - tor_assert(env->key); - - if (crypto_pk_private_ok(env)) { - new_key = RSAPrivateKey_dup(env->key); - privatekey = 1; - } else { - new_key = RSAPublicKey_dup(env->key); - } - if (!new_key) { - /* LCOV_EXCL_START - * - * We can't cause RSA*Key_dup() to fail, so we can't really test this. - */ - log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.", - privatekey?"private":"public"); - crypto_log_errors(LOG_ERR, - privatekey ? "Duplicating a private key" : - "Duplicating a public key"); - tor_fragile_assert(); - return NULL; - /* LCOV_EXCL_STOP */ - } - - return crypto_new_pk_from_rsa_(new_key); -} - -/** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key - * in <b>env</b>, using the padding method <b>padding</b>. On success, - * write the result to <b>to</b>, and return the number of bytes - * written. On failure, return -1. - * - * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be - * at least the length of the modulus of <b>env</b>. - */ -int -crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen, int padding) -{ - int r; - tor_assert(env); - tor_assert(from); - tor_assert(to); - tor_assert(fromlen<INT_MAX); - tor_assert(tolen >= crypto_pk_keysize(env)); - - r = RSA_public_encrypt((int)fromlen, - (unsigned char*)from, (unsigned char*)to, - env->key, crypto_get_rsa_padding(padding)); - if (r<0) { - crypto_log_errors(LOG_WARN, "performing RSA encryption"); - return -1; - } - return r; -} - -/** Decrypt <b>fromlen</b> bytes from <b>from</b> with the private key - * in <b>env</b>, using the padding method <b>padding</b>. On success, - * write the result to <b>to</b>, and return the number of bytes - * written. On failure, return -1. - * - * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be - * at least the length of the modulus of <b>env</b>. - */ -int -crypto_pk_private_decrypt(crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen, - int padding, int warnOnFailure) -{ - int r; - tor_assert(env); - tor_assert(from); - tor_assert(to); - tor_assert(env->key); - tor_assert(fromlen<INT_MAX); - tor_assert(tolen >= crypto_pk_keysize(env)); - if (!crypto_pk_key_is_private(env)) - /* Not a private key */ - return -1; - - r = RSA_private_decrypt((int)fromlen, - (unsigned char*)from, (unsigned char*)to, - env->key, crypto_get_rsa_padding(padding)); - - if (r<0) { - crypto_log_errors(warnOnFailure?LOG_WARN:LOG_DEBUG, - "performing RSA decryption"); - return -1; - } - return r; -} - -/** Check the signature in <b>from</b> (<b>fromlen</b> bytes long) with the - * public key in <b>env</b>, using PKCS1 padding. On success, write the - * signed data to <b>to</b>, and return the number of bytes written. - * On failure, return -1. - * - * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be - * at least the length of the modulus of <b>env</b>. - */ -int -crypto_pk_public_checksig(const crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen) -{ - int r; - tor_assert(env); - tor_assert(from); - tor_assert(to); - tor_assert(fromlen < INT_MAX); - tor_assert(tolen >= crypto_pk_keysize(env)); - r = RSA_public_decrypt((int)fromlen, - (unsigned char*)from, (unsigned char*)to, - env->key, RSA_PKCS1_PADDING); - - if (r<0) { - crypto_log_errors(LOG_INFO, "checking RSA signature"); - return -1; - } - return r; -} - -/** Check a siglen-byte long signature at <b>sig</b> against - * <b>datalen</b> bytes of data at <b>data</b>, using the public key - * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for - * SHA1(data). Else return -1. - */ -int -crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, size_t siglen) -{ - char digest[DIGEST_LEN]; - char *buf; - size_t buflen; - int r; - - tor_assert(env); - tor_assert(data); - tor_assert(sig); - tor_assert(datalen < SIZE_T_CEILING); - tor_assert(siglen < SIZE_T_CEILING); - - if (crypto_digest(digest,data,datalen)<0) { - log_warn(LD_BUG, "couldn't compute digest"); - return -1; - } - buflen = crypto_pk_keysize(env); - buf = tor_malloc(buflen); - r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen); - if (r != DIGEST_LEN) { - log_warn(LD_CRYPTO, "Invalid signature"); - tor_free(buf); - return -1; - } - if (tor_memneq(buf, digest, DIGEST_LEN)) { - log_warn(LD_CRYPTO, "Signature mismatched with digest."); - tor_free(buf); - return -1; - } - tor_free(buf); - - return 0; -} - -/** Sign <b>fromlen</b> bytes of data from <b>from</b> with the private key in - * <b>env</b>, using PKCS1 padding. On success, write the signature to - * <b>to</b>, and return the number of bytes written. On failure, return - * -1. - * - * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be - * at least the length of the modulus of <b>env</b>. - */ -int -crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen) -{ - int r; - tor_assert(env); - tor_assert(from); - tor_assert(to); - tor_assert(fromlen < INT_MAX); - tor_assert(tolen >= crypto_pk_keysize(env)); - if (!crypto_pk_key_is_private(env)) - /* Not a private key */ - return -1; - - r = RSA_private_encrypt((int)fromlen, - (unsigned char*)from, (unsigned char*)to, - (RSA*)env->key, RSA_PKCS1_PADDING); - if (r<0) { - crypto_log_errors(LOG_WARN, "generating RSA signature"); - return -1; - } - return r; -} - -/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at - * <b>from</b>; sign the data with the private key in <b>env</b>, and - * store it in <b>to</b>. Return the number of bytes written on - * success, and -1 on failure. - * - * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be - * at least the length of the modulus of <b>env</b>. - */ -int -crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen) -{ - int r; - char digest[DIGEST_LEN]; - if (crypto_digest(digest,from,fromlen)<0) - return -1; - r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN); - memwipe(digest, 0, sizeof(digest)); - return r; -} - -/** Perform a hybrid (public/secret) encryption on <b>fromlen</b> - * bytes of data from <b>from</b>, with padding type 'padding', - * storing the results on <b>to</b>. - * - * Returns the number of bytes written on success, -1 on failure. - * - * The encrypted data consists of: - * - The source data, padded and encrypted with the public key, if the - * padded source data is no longer than the public key, and <b>force</b> - * is false, OR - * - The beginning of the source data prefixed with a 16-byte symmetric key, - * padded and encrypted with the public key; followed by the rest of - * the source data encrypted in AES-CTR mode with the symmetric key. - */ -int -crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, - char *to, size_t tolen, - const char *from, - size_t fromlen, - int padding, int force) -{ - int overhead, outlen, r; - size_t pkeylen, symlen; - crypto_cipher_t *cipher = NULL; - char *buf = NULL; - - tor_assert(env); - tor_assert(from); - tor_assert(to); - tor_assert(fromlen < SIZE_T_CEILING); - - overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding)); - pkeylen = crypto_pk_keysize(env); - - if (!force && fromlen+overhead <= pkeylen) { - /* It all fits in a single encrypt. */ - return crypto_pk_public_encrypt(env,to, - tolen, - from,fromlen,padding); - } - tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN); - tor_assert(tolen >= pkeylen); - - char key[CIPHER_KEY_LEN]; - crypto_rand(key, sizeof(key)); /* generate a new key. */ - cipher = crypto_cipher_new(key); - - buf = tor_malloc(pkeylen+1); - memcpy(buf, key, CIPHER_KEY_LEN); - memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN); - - /* Length of symmetrically encrypted data. */ - symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN); - - outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding); - if (outlen!=(int)pkeylen) { - goto err; - } - r = crypto_cipher_encrypt(cipher, to+outlen, - from+pkeylen-overhead-CIPHER_KEY_LEN, symlen); - - if (r<0) goto err; - memwipe(buf, 0, pkeylen); - memwipe(key, 0, sizeof(key)); - tor_free(buf); - crypto_cipher_free(cipher); - tor_assert(outlen+symlen < INT_MAX); - return (int)(outlen + symlen); - err: - - memwipe(buf, 0, pkeylen); - memwipe(key, 0, sizeof(key)); - tor_free(buf); - crypto_cipher_free(cipher); - return -1; -} - -/** Invert crypto_pk_public_hybrid_encrypt. Returns the number of bytes - * written on success, -1 on failure. */ -int -crypto_pk_private_hybrid_decrypt(crypto_pk_t *env, - char *to, - size_t tolen, - const char *from, - size_t fromlen, - int padding, int warnOnFailure) -{ - int outlen, r; - size_t pkeylen; - crypto_cipher_t *cipher = NULL; - char *buf = NULL; - - tor_assert(fromlen < SIZE_T_CEILING); - pkeylen = crypto_pk_keysize(env); - - if (fromlen <= pkeylen) { - return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding, - warnOnFailure); - } - - buf = tor_malloc(pkeylen); - outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding, - warnOnFailure); - if (outlen<0) { - log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO, - "Error decrypting public-key data"); - goto err; - } - if (outlen < CIPHER_KEY_LEN) { - log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO, - "No room for a symmetric key"); - goto err; - } - cipher = crypto_cipher_new(buf); - if (!cipher) { - goto err; - } - memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN); - outlen -= CIPHER_KEY_LEN; - tor_assert(tolen - outlen >= fromlen - pkeylen); - r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen); - if (r<0) - goto err; - memwipe(buf,0,pkeylen); - tor_free(buf); - crypto_cipher_free(cipher); - tor_assert(outlen + fromlen < INT_MAX); - return (int)(outlen + (fromlen-pkeylen)); - err: - memwipe(buf,0,pkeylen); - tor_free(buf); - crypto_cipher_free(cipher); - return -1; -} - -/** ASN.1-encode the public portion of <b>pk</b> into <b>dest</b>. - * Return -1 on error, or the number of characters used on success. - */ -int -crypto_pk_asn1_encode(crypto_pk_t *pk, char *dest, size_t dest_len) -{ - int len; - unsigned char *buf = NULL; - - len = i2d_RSAPublicKey(pk->key, &buf); - if (len < 0 || buf == NULL) - return -1; - - if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) { - OPENSSL_free(buf); - return -1; - } - /* We don't encode directly into 'dest', because that would be illegal - * type-punning. (C99 is smarter than me, C99 is smarter than me...) - */ - memcpy(dest,buf,len); - OPENSSL_free(buf); - return len; -} - -/** Decode an ASN.1-encoded public key from <b>str</b>; return the result on - * success and NULL on failure. - */ -crypto_pk_t * -crypto_pk_asn1_decode(const char *str, size_t len) -{ - RSA *rsa; - unsigned char *buf; - const unsigned char *cp; - cp = buf = tor_malloc(len); - memcpy(buf,str,len); - rsa = d2i_RSAPublicKey(NULL, &cp, len); - tor_free(buf); - if (!rsa) { - crypto_log_errors(LOG_WARN,"decoding public key"); - return NULL; - } - return crypto_new_pk_from_rsa_(rsa); -} - -/** Given a private or public key <b>pk</b>, put a SHA1 hash of the - * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space). - * Return 0 on success, -1 on failure. - */ -int -crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out) -{ - unsigned char *buf = NULL; - int len; - - len = i2d_RSAPublicKey((RSA*)pk->key, &buf); - if (len < 0 || buf == NULL) - return -1; - if (crypto_digest(digest_out, (char*)buf, len) < 0) { - OPENSSL_free(buf); - return -1; - } - OPENSSL_free(buf); - return 0; -} - -/** Compute all digests of the DER encoding of <b>pk</b>, and store them - * in <b>digests_out</b>. Return 0 on success, -1 on failure. */ -int -crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out) -{ - unsigned char *buf = NULL; - int len; - - len = i2d_RSAPublicKey(pk->key, &buf); - if (len < 0 || buf == NULL) - return -1; - if (crypto_common_digests(digests_out, (char*)buf, len) < 0) { - OPENSSL_free(buf); - return -1; - } - OPENSSL_free(buf); - return 0; -} - /** Copy <b>in</b> to the <b>outlen</b>-byte buffer <b>out</b>, adding spaces * every four characters. */ void @@ -1479,127 +390,6 @@ crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in) *out = '\0'; } -/** Given a private or public key <b>pk</b>, put a fingerprint of the - * public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 bytes of - * space). Return 0 on success, -1 on failure. - * - * Fingerprints are computed as the SHA1 digest of the ASN.1 encoding - * of the public key, converted to hexadecimal, in upper case, with a - * space after every four digits. - * - * If <b>add_space</b> is false, omit the spaces. - */ -int -crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out, int add_space) -{ - char digest[DIGEST_LEN]; - char hexdigest[HEX_DIGEST_LEN+1]; - if (crypto_pk_get_digest(pk, digest)) { - return -1; - } - base16_encode(hexdigest,sizeof(hexdigest),digest,DIGEST_LEN); - if (add_space) { - crypto_add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest); - } else { - strncpy(fp_out, hexdigest, HEX_DIGEST_LEN+1); - } - return 0; -} - -/** Given a private or public key <b>pk</b>, put a hashed fingerprint of - * the public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 - * bytes of space). Return 0 on success, -1 on failure. - * - * Hashed fingerprints are computed as the SHA1 digest of the SHA1 digest - * of the ASN.1 encoding of the public key, converted to hexadecimal, in - * upper case. - */ -int -crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out) -{ - char digest[DIGEST_LEN], hashed_digest[DIGEST_LEN]; - if (crypto_pk_get_digest(pk, digest)) { - return -1; - } - if (crypto_digest(hashed_digest, digest, DIGEST_LEN)) { - return -1; - } - base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN); - return 0; -} - -/** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the - * Base64 encoding of the DER representation of the private key as a NUL - * terminated string, and return it via <b>priv_out</b>. Return 0 on - * sucess, -1 on failure. - * - * It is the caller's responsibility to sanitize and free the resulting buffer. - */ -int -crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out) -{ - unsigned char *der = NULL; - int der_len; - int ret = -1; - - *priv_out = NULL; - - der_len = i2d_RSAPrivateKey(pk->key, &der); - if (der_len < 0 || der == NULL) - return ret; - - size_t priv_len = base64_encode_size(der_len, 0) + 1; - char *priv = tor_malloc_zero(priv_len); - if (base64_encode(priv, priv_len, (char *)der, der_len, 0) >= 0) { - *priv_out = priv; - ret = 0; - } else { - tor_free(priv); - } - - memwipe(der, 0, der_len); - OPENSSL_free(der); - return ret; -} - -/** Given a string containing the Base64 encoded DER representation of the - * private key <b>str</b>, decode and return the result on success, or NULL - * on failure. - */ -crypto_pk_t * -crypto_pk_base64_decode(const char *str, size_t len) -{ - crypto_pk_t *pk = NULL; - - char *der = tor_malloc_zero(len + 1); - int der_len = base64_decode(der, len, str, len); - if (der_len <= 0) { - log_warn(LD_CRYPTO, "Stored RSA private key seems corrupted (base64)."); - goto out; - } - - const unsigned char *dp = (unsigned char*)der; /* Shut the compiler up. */ - RSA *rsa = d2i_RSAPrivateKey(NULL, &dp, der_len); - if (!rsa) { - crypto_log_errors(LOG_WARN, "decoding private key"); - goto out; - } - - pk = crypto_new_pk_from_rsa_(rsa); - - /* Make sure it's valid. */ - if (crypto_pk_check_key(pk) <= 0) { - crypto_pk_free(pk); - pk = NULL; - goto out; - } - - out: - memwipe(der, 0, len + 1); - tor_free(der); - return pk; -} - /* symmetric crypto */ /** Encrypt <b>fromlen</b> bytes from <b>from</b> using the cipher @@ -1711,469 +501,6 @@ crypto_cipher_decrypt_with_iv(const char *key, return (int)(fromlen - CIPHER_IV_LEN); } -/* SHA-1 */ - -/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in - * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. - * Return 0 on success, 1 on failure. - */ -int -crypto_digest(char *digest, const char *m, size_t len) -{ - tor_assert(m); - tor_assert(digest); - return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); -} - -/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result - * into <b>digest</b>. Return 0 on success, 1 on failure. */ -int -crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - if (algorithm == DIGEST_SHA256) - return (SHA256((const uint8_t*)m,len,(uint8_t*)digest) == NULL); - else - return (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - == -1); -} - -/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result - * into <b>digest</b>. Return 0 on success, 1 on failure. */ -int -crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - if (algorithm == DIGEST_SHA512) - return (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - == NULL); - else - return (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - == -1); -} - -/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the - * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on - * success, -1 on failure. */ -int -crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len) -{ - tor_assert(ds_out); - memset(ds_out, 0, sizeof(*ds_out)); - if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) - return -1; - if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0) - return -1; - - return 0; -} - -/** Return the name of an algorithm, as used in directory documents. */ -const char * -crypto_digest_algorithm_get_name(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: - return "sha1"; - case DIGEST_SHA256: - return "sha256"; - case DIGEST_SHA512: - return "sha512"; - case DIGEST_SHA3_256: - return "sha3-256"; - case DIGEST_SHA3_512: - return "sha3-512"; - default: - // LCOV_EXCL_START - tor_fragile_assert(); - return "??unknown_digest??"; - // LCOV_EXCL_STOP - } -} - -/** Given the name of a digest algorithm, return its integer value, or -1 if - * the name is not recognized. */ -int -crypto_digest_algorithm_parse_name(const char *name) -{ - if (!strcmp(name, "sha1")) - return DIGEST_SHA1; - else if (!strcmp(name, "sha256")) - return DIGEST_SHA256; - else if (!strcmp(name, "sha512")) - return DIGEST_SHA512; - else if (!strcmp(name, "sha3-256")) - return DIGEST_SHA3_256; - else if (!strcmp(name, "sha3-512")) - return DIGEST_SHA3_512; - else - return -1; -} - -/** Given an algorithm, return the digest length in bytes. */ -size_t -crypto_digest_algorithm_get_length(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: - return DIGEST_LEN; - case DIGEST_SHA256: - return DIGEST256_LEN; - case DIGEST_SHA512: - return DIGEST512_LEN; - case DIGEST_SHA3_256: - return DIGEST256_LEN; - case DIGEST_SHA3_512: - return DIGEST512_LEN; - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; /* Unreachable */ // LCOV_EXCL_LINE - } -} - -/** Intermediate information about the digest of a stream of data. */ -struct crypto_digest_t { - digest_algorithm_t algorithm; /**< Which algorithm is in use? */ - /** State for the digest we're using. Only one member of the - * union is usable, depending on the value of <b>algorithm</b>. Note also - * that space for other members might not even be allocated! - */ - union { - SHA_CTX sha1; /**< state for SHA1 */ - SHA256_CTX sha2; /**< state for SHA256 */ - SHA512_CTX sha512; /**< state for SHA512 */ - keccak_state sha3; /**< state for SHA3-[256,512] */ - } d; -}; - -/** - * Return the number of bytes we need to malloc in order to get a - * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe - * when we free one. - */ -static size_t -crypto_digest_alloc_bytes(digest_algorithm_t alg) -{ - /* Helper: returns the number of bytes in the 'f' field of 'st' */ -#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) - /* Gives the length of crypto_digest_t through the end of the field 'd' */ -#define END_OF_FIELD(f) (STRUCT_OFFSET(crypto_digest_t, f) + \ - STRUCT_FIELD_SIZE(crypto_digest_t, f)) - switch (alg) { - case DIGEST_SHA1: - return END_OF_FIELD(d.sha1); - case DIGEST_SHA256: - return END_OF_FIELD(d.sha2); - case DIGEST_SHA512: - return END_OF_FIELD(d.sha512); - case DIGEST_SHA3_256: - case DIGEST_SHA3_512: - return END_OF_FIELD(d.sha3); - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; // LCOV_EXCL_LINE - } -#undef END_OF_FIELD -#undef STRUCT_FIELD_SIZE -} - -/** - * Internal function: create and return a new digest object for 'algorithm'. - * Does not typecheck the algorithm. - */ -static crypto_digest_t * -crypto_digest_new_internal(digest_algorithm_t algorithm) -{ - crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - r->algorithm = algorithm; - - switch (algorithm) - { - case DIGEST_SHA1: - SHA1_Init(&r->d.sha1); - break; - case DIGEST_SHA256: - SHA256_Init(&r->d.sha2); - break; - case DIGEST_SHA512: - SHA512_Init(&r->d.sha512); - break; - case DIGEST_SHA3_256: - keccak_digest_init(&r->d.sha3, 256); - break; - case DIGEST_SHA3_512: - keccak_digest_init(&r->d.sha3, 512); - break; - default: - tor_assert_unreached(); - } - - return r; -} - -/** Allocate and return a new digest object to compute SHA1 digests. - */ -crypto_digest_t * -crypto_digest_new(void) -{ - return crypto_digest_new_internal(DIGEST_SHA1); -} - -/** Allocate and return a new digest object to compute 256-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest256_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - return crypto_digest_new_internal(algorithm); -} - -/** Allocate and return a new digest object to compute 512-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest512_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - return crypto_digest_new_internal(algorithm); -} - -/** Deallocate a digest object. - */ -void -crypto_digest_free(crypto_digest_t *digest) -{ - if (!digest) - return; - size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - memwipe(digest, 0, bytes); - tor_free(digest); -} - -/** Add <b>len</b> bytes from <b>data</b> to the digest object. - */ -void -crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len) -{ - tor_assert(digest); - tor_assert(data); - /* Using the SHA*_*() calls directly means we don't support doing - * SHA in hardware. But so far the delay of getting the question - * to the hardware, and hearing the answer, is likely higher than - * just doing it ourselves. Hashes are fast. - */ - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Update(&digest->d.sha1, (void*)data, len); - break; - case DIGEST_SHA256: - SHA256_Update(&digest->d.sha2, (void*)data, len); - break; - case DIGEST_SHA512: - SHA512_Update(&digest->d.sha512, (void*)data, len); - break; - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); - break; - default: - /* LCOV_EXCL_START */ - tor_fragile_assert(); - break; - /* LCOV_EXCL_STOP */ - } -} - -/** Compute the hash of the data that has been passed to the digest - * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. - */ -void -crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len) -{ - unsigned char r[DIGEST512_LEN]; - crypto_digest_t tmpenv; - tor_assert(digest); - tor_assert(out); - tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); - - /* The SHA-3 code handles copying into a temporary ctx, and also can handle - * short output buffers by truncating appropriately. */ - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { - keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); - return; - } - - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - /* memcpy into a temporary ctx, since SHA*_Final clears the context */ - memcpy(&tmpenv, digest, alloc_bytes); - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Final(r, &tmpenv.d.sha1); - break; - case DIGEST_SHA256: - SHA256_Final(r, &tmpenv.d.sha2); - break; - case DIGEST_SHA512: - SHA512_Final(r, &tmpenv.d.sha512); - break; -//LCOV_EXCL_START - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - default: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - /* This is fatal, because it should never happen. */ - tor_assert_unreached(); - break; -//LCOV_EXCL_STOP - } - memcpy(out, r, out_len); - memwipe(r, 0, sizeof(r)); -} - -/** Allocate and return a new digest object with the same state as - * <b>digest</b> - */ -crypto_digest_t * -crypto_digest_dup(const crypto_digest_t *digest) -{ - tor_assert(digest); - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - return tor_memdup(digest, alloc_bytes); -} - -/** Replace the state of the digest object <b>into</b> with the state - * of the digest object <b>from</b>. Requires that 'into' and 'from' - * have the same digest type. - */ -void -crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from) -{ - tor_assert(into); - tor_assert(from); - tor_assert(into->algorithm == from->algorithm); - const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); - memcpy(into,from,alloc_bytes); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of those strings, - * plus the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist(char *digest_out, size_t len_out, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of: the - * optional string <b>prepend</b>, those strings, - * and the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>len_out</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_t *d = crypto_digest_new_internal(alg); - if (prepend) - crypto_digest_add_bytes(d, prepend, strlen(prepend)); - SMARTLIST_FOREACH(lst, const char *, cp, - crypto_digest_add_bytes(d, cp, strlen(cp))); - if (append) - crypto_digest_add_bytes(d, append, strlen(append)); - crypto_digest_get_digest(d, digest_out, len_out); - crypto_digest_free(d); -} - -/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using - * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte - * result in <b>hmac_out</b>. Asserts on failure. - */ -void -crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len) -{ - unsigned char *rv = NULL; - /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ - tor_assert(key_len < INT_MAX); - tor_assert(msg_len < INT_MAX); - tor_assert(hmac_out); - rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, - (unsigned char*)hmac_out, NULL); - tor_assert(rv); -} - -/** Internal state for a eXtendable-Output Function (XOF). */ -struct crypto_xof_t { - keccak_state s; -}; - -/** Allocate a new XOF object backed by SHAKE-256. The security level - * provided is a function of the length of the output used. Read and - * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output - * Functions" before using this construct. - */ -crypto_xof_t * -crypto_xof_new(void) -{ - crypto_xof_t *xof; - xof = tor_malloc(sizeof(crypto_xof_t)); - keccak_xof_init(&xof->s, 256); - return xof; -} - -/** Absorb bytes into a XOF object. Must not be called after a call to - * crypto_xof_squeeze_bytes() for the same instance, and will assert - * if attempted. - */ -void -crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) -{ - int i = keccak_xof_absorb(&xof->s, data, len); - tor_assert(i == 0); -} - -/** Squeeze bytes out of a XOF object. Calling this routine will render - * the XOF instance ineligible to absorb further data. - */ -void -crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) -{ - int i = keccak_xof_squeeze(&xof->s, out, len); - tor_assert(i == 0); -} - -/** Cleanse and deallocate a XOF object. */ -void -crypto_xof_free(crypto_xof_t *xof) -{ - if (!xof) - return; - memwipe(xof, 0, sizeof(crypto_xof_t)); - tor_free(xof); -} - /* DH */ /** Our DH 'g' parameter */ @@ -2207,12 +534,12 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) goto out; if (!DH_set0_pqg(dh, dh_p, NULL, dh_g)) goto out; -#else +#else /* !(defined(OPENSSL_1_1_API)) */ if (!(dh->p = BN_dup(p))) goto out; if (!(dh->g = BN_dup(g))) goto out; -#endif +#endif /* defined(OPENSSL_1_1_API) */ /* Perform the validation. */ int codes = 0; @@ -2383,7 +710,7 @@ crypto_dh_new(int dh_type) if (!DH_set_length(res->dh, DH_PRIVATE_KEY_BITS)) goto err; -#else +#else /* !(defined(OPENSSL_1_1_API)) */ if (dh_type == DH_TYPE_TLS) { if (!(res->dh->p = BN_dup(dh_param_p_tls))) goto err; @@ -2396,12 +723,13 @@ crypto_dh_new(int dh_type) goto err; res->dh->length = DH_PRIVATE_KEY_BITS; -#endif +#endif /* defined(OPENSSL_1_1_API) */ return res; - err: + /* LCOV_EXCL_START * This error condition is only reached when an allocation fails */ + err: crypto_log_errors(LOG_WARN, "creating DH object"); if (res->dh) DH_free(res->dh); /* frees p and g too */ tor_free(res); @@ -2458,7 +786,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) "the-universe chances really do happen. Treating as a failure."); return -1; } -#else +#else /* !(defined(OPENSSL_1_1_API)) */ if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { /* LCOV_EXCL_START * If this happens, then openssl's DH implementation is busted. */ @@ -2471,7 +799,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) goto again; /* LCOV_EXCL_STOP */ } -#endif +#endif /* defined(OPENSSL_1_1_API) */ return 0; } @@ -2492,7 +820,7 @@ crypto_dh_get_public(crypto_dh_t *dh, char *pubkey, size_t pubkey_len) DH_get0_key(dh->dh, &dh_pub, &dh_priv); #else dh_pub = dh->dh->pub_key; -#endif +#endif /* defined(OPENSSL_1_1_API) */ if (!dh_pub) { if (crypto_dh_generate_public(dh)<0) @@ -2643,7 +971,7 @@ crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len, for (cp = key_out, i=0; cp < key_out+key_out_len; ++i, cp += DIGEST_LEN) { tmp[key_in_len] = i; - if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1)) + if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1) < 0) goto exit; memcpy(cp, digest, MIN(DIGEST_LEN, key_out_len-(cp-key_out))); } @@ -2718,7 +1046,7 @@ crypto_expand_key_material_rfc5869_sha256( /** Free a DH key exchange object. */ void -crypto_dh_free(crypto_dh_t *dh) +crypto_dh_free_(crypto_dh_t *dh) { if (!dh) return; @@ -2727,673 +1055,22 @@ crypto_dh_free(crypto_dh_t *dh) tor_free(dh); } -/* random numbers */ - -/** How many bytes of entropy we add at once. - * - * This is how much entropy OpenSSL likes to add right now, so maybe it will - * work for us too. */ -#define ADD_ENTROPY 32 - -/** Set the seed of the weak RNG to a random value. */ -void -crypto_seed_weak_rng(tor_weak_rng_t *rng) -{ - unsigned seed; - crypto_rand((void*)&seed, sizeof(seed)); - tor_init_weak_random(rng, seed); -} - -#ifdef TOR_UNIT_TESTS -int break_strongest_rng_syscall = 0; -int break_strongest_rng_fallback = 0; -#endif - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on - * failure. A maximum request size of 256 bytes is imposed. - */ -static int -crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) -{ - tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); - -#ifdef TOR_UNIT_TESTS - if (break_strongest_rng_syscall) - return -1; -#endif - -#if defined(_WIN32) - static int provider_set = 0; - static HCRYPTPROV provider; - - if (!provider_set) { - if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT)) { - log_warn(LD_CRYPTO, "Can't get CryptoAPI provider [1]"); - return -1; - } - provider_set = 1; - } - if (!CryptGenRandom(provider, out_len, out)) { - log_warn(LD_CRYPTO, "Can't get entropy from CryptoAPI."); - return -1; - } - - return 0; -#elif defined(__linux__) && defined(SYS_getrandom) - static int getrandom_works = 1; /* Be optimitic about our chances... */ - - /* getrandom() isn't as straight foward as getentropy(), and has - * no glibc wrapper. - * - * As far as I can tell from getrandom(2) and the source code, the - * requests we issue will always succeed (though it will block on the - * call if /dev/urandom isn't seeded yet), since we are NOT specifying - * GRND_NONBLOCK and the request is <= 256 bytes. - * - * The manpage is unclear on what happens if a signal interrupts the call - * while the request is blocked due to lack of entropy.... - * - * We optimistically assume that getrandom() is available and functional - * because it is the way of the future, and 2 branch mispredicts pale in - * comparision to the overheads involved with failing to open - * /dev/srandom followed by opening and reading from /dev/urandom. - */ - if (PREDICT_LIKELY(getrandom_works)) { - long ret; - /* A flag of '0' here means to read from '/dev/urandom', and to - * block if insufficient entropy is available to service the - * request. - */ - const unsigned int flags = 0; - do { - ret = syscall(SYS_getrandom, out, out_len, flags); - } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); - - if (PREDICT_UNLIKELY(ret == -1)) { - /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ - tor_assert(errno != EAGAIN); - tor_assert(errno != EINTR); - - /* Probably ENOSYS. */ - log_warn(LD_CRYPTO, "Can't get entropy from getrandom()."); - getrandom_works = 0; /* Don't bother trying again. */ - return -1; - /* LCOV_EXCL_STOP */ - } - - tor_assert(ret == (long)out_len); - return 0; - } - - return -1; /* getrandom() previously failed unexpectedly. */ -#elif defined(HAVE_GETENTROPY) - /* getentropy() is what Linux's getrandom() wants to be when it grows up. - * the only gotcha is that requests are limited to 256 bytes. - */ - return getentropy(out, out_len); -#else - (void) out; -#endif - - /* This platform doesn't have a supported syscall based random. */ - return -1; -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * via the per-platform fallback mechanism, storing it into <b>out</b>. - * Return 0 on success, -1 on failure. A maximum request size of 256 bytes - * is imposed. - */ -static int -crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) -{ -#ifdef TOR_UNIT_TESTS - if (break_strongest_rng_fallback) - return -1; -#endif - -#ifdef _WIN32 - /* Windows exclusively uses crypto_strongest_rand_syscall(). */ - (void)out; - (void)out_len; - return -1; -#else - static const char *filenames[] = { - "/dev/srandom", "/dev/urandom", "/dev/random", NULL - }; - int fd, i; - size_t n; - - for (i = 0; filenames[i]; ++i) { - log_debug(LD_FS, "Opening %s for entropy", filenames[i]); - fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); - if (fd<0) continue; - log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); - n = read_all(fd, (char*)out, out_len, 0); - close(fd); - if (n != out_len) { - /* LCOV_EXCL_START - * We can't make /dev/foorandom actually fail. */ - log_warn(LD_CRYPTO, - "Error reading from entropy source (read only %lu bytes).", - (unsigned long)n); - return -1; - /* LCOV_EXCL_STOP */ - } - - return 0; - } - - return -1; -#endif -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum - * request size of 256 bytes is imposed. - */ -STATIC int -crypto_strongest_rand_raw(uint8_t *out, size_t out_len) -{ - static const size_t sanity_min_size = 16; - static const int max_attempts = 3; - tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); - - /* For buffers >= 16 bytes (128 bits), we sanity check the output by - * zero filling the buffer and ensuring that it actually was at least - * partially modified. - * - * Checking that any individual byte is non-zero seems like it would - * fail too often (p = out_len * 1/256) for comfort, but this is an - * "adjust according to taste" sort of check. - */ - memwipe(out, 0, out_len); - for (int i = 0; i < max_attempts; i++) { - /* Try to use the syscall/OS favored mechanism to get strong entropy. */ - if (crypto_strongest_rand_syscall(out, out_len) != 0) { - /* Try to use the less-favored mechanism to get strong entropy. */ - if (crypto_strongest_rand_fallback(out, out_len) != 0) { - /* Welp, we tried. Hopefully the calling code terminates the process - * since we're basically boned without good entropy. - */ - log_warn(LD_CRYPTO, - "Cannot get strong entropy: no entropy source found."); - return -1; - } - } - - if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) - return 0; - } - - /* LCOV_EXCL_START - * - * We tried max_attempts times to fill a buffer >= 128 bits long, - * and each time it returned all '0's. Either the system entropy - * source is busted, or the user should go out and buy a ticket to - * every lottery on the planet. - */ - log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); - - return -1; - /* LCOV_EXCL_STOP */ -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * storing it into <b>out</b>. - */ -void -crypto_strongest_rand(uint8_t *out, size_t out_len) -{ -#define DLEN SHA512_DIGEST_LENGTH - /* We're going to hash DLEN bytes from the system RNG together with some - * bytes from the openssl PRNG, in order to yield DLEN bytes. - */ - uint8_t inp[DLEN*2]; - uint8_t tmp[DLEN]; - tor_assert(out); - while (out_len) { - crypto_rand((char*) inp, DLEN); - if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { - // LCOV_EXCL_START - log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " - "important key. Exiting."); - /* Die with an assertion so we get a stack trace. */ - tor_assert(0); - // LCOV_EXCL_STOP - } - if (out_len >= DLEN) { - SHA512(inp, sizeof(inp), out); - out += DLEN; - out_len -= DLEN; - } else { - SHA512(inp, sizeof(inp), tmp); - memcpy(out, tmp, out_len); - break; - } - } - memwipe(tmp, 0, sizeof(tmp)); - memwipe(inp, 0, sizeof(inp)); -#undef DLEN -} - -/** Seed OpenSSL's random number generator with bytes from the operating - * system. Return 0 on success, -1 on failure. - */ -int -crypto_seed_rng(void) -{ - int rand_poll_ok = 0, load_entropy_ok = 0; - uint8_t buf[ADD_ENTROPY]; - - /* OpenSSL has a RAND_poll function that knows about more kinds of - * entropy than we do. We'll try calling that, *and* calling our own entropy - * functions. If one succeeds, we'll accept the RNG as seeded. */ - rand_poll_ok = RAND_poll(); - if (rand_poll_ok == 0) - log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE - - load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); - if (load_entropy_ok) { - RAND_seed(buf, sizeof(buf)); - } - - memwipe(buf, 0, sizeof(buf)); - - if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1) - return 0; - else - return -1; -} - -/** Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking - * for unit tests. - * - * This function is not allowed to fail; if it would fail to generate strong - * entropy, it must terminate the process instead. - */ -MOCK_IMPL(void, -crypto_rand, (char *to, size_t n)) -{ - crypto_rand_unmocked(to, n); -} - -/** Write <b>n</b> bytes of strong random data to <b>to</b>. Most callers - * will want crypto_rand instead. - * - * This function is not allowed to fail; if it would fail to generate strong - * entropy, it must terminate the process instead. - */ -void -crypto_rand_unmocked(char *to, size_t n) -{ - int r; - if (n == 0) - return; - - tor_assert(n < INT_MAX); - tor_assert(to); - r = RAND_bytes((unsigned char*)to, (int)n); - /* We consider a PRNG failure non-survivable. Let's assert so that we get a - * stack trace about where it happened. - */ - tor_assert(r >= 0); -} - -/** Return a pseudorandom integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and - * INT_MAX+1, inclusive. */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * <b>min</b> MUST be in range [0, <b>max</b>). - * <b>max</b> MUST be in range (min, INT_MAX]. - */ -int -crypto_rand_int_range(unsigned int min, unsigned int max) -{ - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** As crypto_rand_int_range, but supports uint64_t. */ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) -{ - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** As crypto_rand_int_range, but supports time_t. */ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. */ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - */ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif - return ((double)u) / UINT_MAX_AS_DOUBLE; -} - -/** Generate and return a new random hostname starting with <b>prefix</b>, - * ending with <b>suffix</b>, and containing no fewer than - * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 - * characters. Does not check for failure. - * - * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE. - **/ -char * -crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, - const char *suffix) -{ - char *result, *rand_bytes; - int randlen, rand_bytes_len; - size_t resultlen, prefixlen; - - if (max_rand_len > MAX_DNS_LABEL_SIZE) - max_rand_len = MAX_DNS_LABEL_SIZE; - if (min_rand_len > max_rand_len) - min_rand_len = max_rand_len; - - randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1); - - prefixlen = strlen(prefix); - resultlen = prefixlen + strlen(suffix) + randlen + 16; - - rand_bytes_len = ((randlen*5)+7)/8; - if (rand_bytes_len % 5) - rand_bytes_len += 5 - (rand_bytes_len%5); - rand_bytes = tor_malloc(rand_bytes_len); - crypto_rand(rand_bytes, rand_bytes_len); - - result = tor_malloc(resultlen); - memcpy(result, prefix, prefixlen); - base32_encode(result+prefixlen, resultlen-prefixlen, - rand_bytes, rand_bytes_len); - tor_free(rand_bytes); - strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen)); - - return result; -} - -/** Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b> - * is empty. */ -void * -smartlist_choose(const smartlist_t *sl) -{ - int len = smartlist_len(sl); - if (len) - return smartlist_get(sl,crypto_rand_int(len)); - return NULL; /* no elements to choose from */ -} - -/** Scramble the elements of <b>sl</b> into a random order. */ -void -smartlist_shuffle(smartlist_t *sl) -{ - int i; - /* From the end of the list to the front, choose at random from the - positions we haven't looked at yet, and swap that position into the - current position. Remember to give "no swap" the same probability as - any other swap. */ - for (i = smartlist_len(sl)-1; i > 0; --i) { - int j = crypto_rand_int(i+1); - smartlist_swap(sl, i, j); - } -} - -/** - * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to - * the value <b>byte</b>. - * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens. - * - * This function is preferable to memset, since many compilers will happily - * optimize out memset() when they can convince themselves that the data being - * cleared will never be read. - * - * Right now, our convention is to use this function when we are wiping data - * that's about to become inaccessible, such as stack buffers that are about - * to go out of scope or structures that are about to get freed. (In - * practice, it appears that the compilers we're currently using will optimize - * out the memset()s for stack-allocated buffers, but not those for - * about-to-be-freed structures. That could change, though, so we're being - * wary.) If there are live reads for the data, then you can just use - * memset(). - */ -void -memwipe(void *mem, uint8_t byte, size_t sz) -{ - if (sz == 0) { - return; - } - /* If sz is nonzero, then mem must not be NULL. */ - tor_assert(mem != NULL); - - /* Data this large is likely to be an underflow. */ - tor_assert(sz < SIZE_T_CEILING); - - /* Because whole-program-optimization exists, we may not be able to just - * have this function call "memset". A smart compiler could inline it, then - * eliminate dead memsets, and declare itself to be clever. */ - -#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) - /* Here's what you do on windows. */ - SecureZeroMemory(mem,sz); -#elif defined(HAVE_RTLSECUREZEROMEMORY) - RtlSecureZeroMemory(mem,sz); -#elif defined(HAVE_EXPLICIT_BZERO) - /* The BSDs provide this. */ - explicit_bzero(mem, sz); -#elif defined(HAVE_MEMSET_S) - /* This is in the C99 standard. */ - memset_s(mem, sz, 0, sz); -#else - /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk - * based on the pointer value, then uses that junk to update a global - * variable. It's an elaborate ruse to trick the compiler into not - * optimizing out the "wipe this memory" code. Read it if you like zany - * programming tricks! In later versions of Tor, we should look for better - * not-optimized-out memory wiping stuff... - * - * ...or maybe not. In practice, there are pure-asm implementations of - * OPENSSL_cleanse() on most platforms, which ought to do the job. - **/ - - OPENSSL_cleanse(mem, sz); -#endif - - /* Just in case some caller of memwipe() is relying on getting a buffer - * filled with a particular value, fill the buffer. - * - * If this function gets inlined, this memset might get eliminated, but - * that's okay: We only care about this particular memset in the case where - * the caller should have been using memset(), and the memset() wouldn't get - * eliminated. In other words, this is here so that we won't break anything - * if somebody accidentally calls memwipe() instead of memset(). - **/ - memset(mem, byte, sz); -} - -#ifndef OPENSSL_THREADS -#error OpenSSL has been built without thread support. Tor requires an \ - OpenSSL library with thread support enabled. -#endif - -#ifndef NEW_THREAD_API -/** Helper: OpenSSL uses this callback to manipulate mutexes. */ -static void -openssl_locking_cb_(int mode, int n, const char *file, int line) -{ - (void)file; - (void)line; - if (!openssl_mutexes_) - /* This is not a really good fix for the - * "release-freed-lock-from-separate-thread-on-shutdown" problem, but - * it can't hurt. */ - return; - if (mode & CRYPTO_LOCK) - tor_mutex_acquire(openssl_mutexes_[n]); - else - tor_mutex_release(openssl_mutexes_[n]); -} - -static void -tor_set_openssl_thread_id(CRYPTO_THREADID *threadid) -{ - CRYPTO_THREADID_set_numeric(threadid, tor_get_thread_id()); -} -#endif - -#if 0 -/* This code is disabled, because OpenSSL never actually uses these callbacks. - */ - -/** OpenSSL helper type: wraps a Tor mutex so that OpenSSL can use it - * as a lock. */ -struct CRYPTO_dynlock_value { - tor_mutex_t *lock; -}; - -/** OpenSSL callback function to allocate a lock: see CRYPTO_set_dynlock_* - * documentation in OpenSSL's docs for more info. */ -static struct CRYPTO_dynlock_value * -openssl_dynlock_create_cb_(const char *file, int line) -{ - struct CRYPTO_dynlock_value *v; - (void)file; - (void)line; - v = tor_malloc(sizeof(struct CRYPTO_dynlock_value)); - v->lock = tor_mutex_new(); - return v; -} - -/** OpenSSL callback function to acquire or release a lock: see - * CRYPTO_set_dynlock_* documentation in OpenSSL's docs for more info. */ -static void -openssl_dynlock_lock_cb_(int mode, struct CRYPTO_dynlock_value *v, - const char *file, int line) -{ - (void)file; - (void)line; - if (mode & CRYPTO_LOCK) - tor_mutex_acquire(v->lock); - else - tor_mutex_release(v->lock); -} - -/** OpenSSL callback function to free a lock: see CRYPTO_set_dynlock_* - * documentation in OpenSSL's docs for more info. */ -static void -openssl_dynlock_destroy_cb_(struct CRYPTO_dynlock_value *v, - const char *file, int line) -{ - (void)file; - (void)line; - tor_mutex_free(v->lock); - tor_free(v); -} -#endif - /** @{ */ -/** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being - * multithreaded. Returns 0. */ -static int -setup_openssl_threading(void) -{ -#ifndef NEW_THREAD_API - int i; - int n = CRYPTO_num_locks(); - n_openssl_mutexes_ = n; - openssl_mutexes_ = tor_calloc(n, sizeof(tor_mutex_t *)); - for (i=0; i < n; ++i) - openssl_mutexes_[i] = tor_mutex_new(); - CRYPTO_set_locking_callback(openssl_locking_cb_); - CRYPTO_THREADID_set_callback(tor_set_openssl_thread_id); -#endif -#if 0 - CRYPTO_set_dynlock_create_callback(openssl_dynlock_create_cb_); - CRYPTO_set_dynlock_lock_callback(openssl_dynlock_lock_cb_); - CRYPTO_set_dynlock_destroy_callback(openssl_dynlock_destroy_cb_); -#endif - return 0; -} - /** Uninitialize the crypto library. Return 0 on success. Does not detect * failure. */ int crypto_global_cleanup(void) { +#ifndef OPENSSL_1_1_API EVP_cleanup(); +#endif #ifndef NEW_THREAD_API ERR_remove_thread_state(NULL); #endif +#ifndef OPENSSL_1_1_API ERR_free_strings(); +#endif if (dh_param_p) BN_clear_free(dh_param_p); @@ -3402,31 +1079,40 @@ crypto_global_cleanup(void) if (dh_param_g) BN_clear_free(dh_param_g); + dh_param_p = dh_param_p_tls = dh_param_g = NULL; + #ifndef DISABLE_ENGINES +#ifndef OPENSSL_1_1_API ENGINE_cleanup(); #endif +#endif CONF_modules_unload(1); +#ifndef OPENSSL_1_1_API CRYPTO_cleanup_all_ex_data(); - -#ifndef NEW_THREAD_API - if (n_openssl_mutexes_) { - int n = n_openssl_mutexes_; - tor_mutex_t **ms = openssl_mutexes_; - int i; - openssl_mutexes_ = NULL; - n_openssl_mutexes_ = 0; - for (i=0;i<n;++i) { - tor_mutex_free(ms[i]); - } - tor_free(ms); - } #endif - tor_free(crypto_openssl_version_str); - tor_free(crypto_openssl_header_version_str); + crypto_openssl_free_all(); + + crypto_early_initialized_ = 0; + crypto_global_initialized_ = 0; + have_seeded_siphash = 0; + siphash_unset_global_key(); + return 0; } /** @} */ +#ifdef USE_DMALLOC +/** Tell the crypto library to use Tor's allocation functions rather than + * calling libc's allocation functions directly. Return 0 on success, -1 + * on failure. */ +int +crypto_use_tor_alloc_functions(void) +{ + int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_); + return r ? 0 : -1; +} +#endif /* defined(USE_DMALLOC) */ + diff --git a/src/common/crypto.h b/src/common/crypto.h index 116e0a62fd..c773557310 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,125 +17,40 @@ #include <stdio.h> #include "torint.h" -#include "testsupport.h" #include "compat.h" +#include "util.h" +#include "crypto_rsa.h" -/* - Macro to create an arbitrary OpenSSL version number as used by - OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard - to read. - - Don't use this directly, instead use one of the other OPENSSL_V macros - below. - - The format is: 4 bits major, 8 bits minor, 8 bits fix, 8 bits patch, 4 bit - status. - */ -#define OPENSSL_VER(a,b,c,d,e) \ - (((a)<<28) | \ - ((b)<<20) | \ - ((c)<<12) | \ - ((d)<< 4) | \ - (e)) -/** An openssl release number. For example, OPENSSL_V(0,9,8,'j') is the - * version for the released version of 0.9.8j */ -#define OPENSSL_V(a,b,c,d) \ - OPENSSL_VER((a),(b),(c),(d)-'a'+1,0xf) -/** An openssl release number for the first release in the series. For - * example, OPENSSL_V_NOPATCH(1,0,0) is the first released version of OpenSSL - * 1.0.0. */ -#define OPENSSL_V_NOPATCH(a,b,c) \ - OPENSSL_VER((a),(b),(c),0,0xf) -/** The first version that would occur for any alpha or beta in an openssl - * series. For example, OPENSSL_V_SERIES(0,9,8) is greater than any released - * 0.9.7, and less than any released 0.9.8. */ -#define OPENSSL_V_SERIES(a,b,c) \ - OPENSSL_VER((a),(b),(c),0,0) - -/** Length of the output of our message digest. */ -#define DIGEST_LEN 20 -/** Length of the output of our second (improved) message digests. (For now - * this is just sha256, but it could be any other 256-bit digest.) */ -#define DIGEST256_LEN 32 -/** Length of the output of our 64-bit optimized message digests (SHA512). */ -#define DIGEST512_LEN 64 -/** Length of our symmetric cipher's keys. */ +/** Length of our symmetric cipher's keys of 128-bit. */ #define CIPHER_KEY_LEN 16 -/** Length of our symmetric cipher's IV. */ +/** Length of our symmetric cipher's IV of 128-bit. */ #define CIPHER_IV_LEN 16 -/** Length of our public keys. */ -#define PK_BYTES (1024/8) +/** Length of our symmetric cipher's keys of 256-bit. */ +#define CIPHER256_KEY_LEN 32 /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a sha1 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST_LEN 27 -/** Length of a sha256 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST256_LEN 43 -/** Length of a sha512 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST512_LEN 86 - -/** Constant used to indicate OAEP padding for public-key encryption */ -#define PK_PKCS1_OAEP_PADDING 60002 - -/** Number of bytes added for PKCS1-OAEP padding. */ -#define PKCS1_OAEP_PADDING_OVERHEAD 42 - /** Length of encoded public key fingerprints, including space; but not * including terminating NUL. */ #define FINGERPRINT_LEN 49 -/** Length of hex encoding of SHA1 digest, not including final NUL. */ -#define HEX_DIGEST_LEN 40 -/** Length of hex encoding of SHA256 digest, not including final NUL. */ -#define HEX_DIGEST256_LEN 64 -/** Length of hex encoding of SHA512 digest, not including final NUL. */ -#define HEX_DIGEST512_LEN 128 - -typedef enum { - DIGEST_SHA1 = 0, - DIGEST_SHA256 = 1, - DIGEST_SHA512 = 2, - DIGEST_SHA3_256 = 3, - DIGEST_SHA3_512 = 4, -} digest_algorithm_t; -#define N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1) -#define N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1) -/** A set of all the digests we commonly compute, taken on a single - * string. Any digests that are shorter than 512 bits are right-padded - * with 0 bits. - * - * Note that this representation wastes 44 bytes for the SHA1 case, so - * don't use it for anything where we need to allocate a whole bunch at - * once. - **/ -typedef struct { - char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN]; -} common_digests_t; - -typedef struct crypto_pk_t crypto_pk_t; typedef struct aes_cnt_cipher crypto_cipher_t; -typedef struct crypto_digest_t crypto_digest_t; -typedef struct crypto_xof_t crypto_xof_t; typedef struct crypto_dh_t crypto_dh_t; /* global state */ -const char * crypto_openssl_get_version_str(void); -const char * crypto_openssl_get_header_version_str(void); +int crypto_init_siphash_key(void); int crypto_early_init(void) ATTR_WUR; int crypto_global_init(int hardwareAccel, const char *accelName, const char *accelPath) ATTR_WUR; +#ifdef USE_DMALLOC +int crypto_use_tor_alloc_functions(void); +#endif + void crypto_thread_cleanup(void); int crypto_global_cleanup(void); /* environment setup */ -MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void)); -void crypto_pk_free(crypto_pk_t *env); - void crypto_set_tls_dh_prime(void); crypto_cipher_t *crypto_cipher_new(const char *key); crypto_cipher_t *crypto_cipher_new_with_bits(const char *key, int bits); @@ -143,68 +58,9 @@ crypto_cipher_t *crypto_cipher_new_with_iv(const char *key, const char *iv); crypto_cipher_t *crypto_cipher_new_with_iv_and_bits(const uint8_t *key, const uint8_t *iv, int bits); -void crypto_cipher_free(crypto_cipher_t *env); - -/* public key crypto */ -MOCK_DECL(int, crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)); -#define crypto_pk_generate_key(env) \ - crypto_pk_generate_key_with_bits((env), (PK_BYTES*8)) - -int crypto_pk_read_private_key_from_filename(crypto_pk_t *env, - const char *keyfile); -int crypto_pk_write_public_key_to_string(crypto_pk_t *env, - char **dest, size_t *len); -int crypto_pk_write_private_key_to_string(crypto_pk_t *env, - char **dest, size_t *len); -int crypto_pk_read_public_key_from_string(crypto_pk_t *env, - const char *src, size_t len); -int crypto_pk_read_private_key_from_string(crypto_pk_t *env, - const char *s, ssize_t len); -int crypto_pk_write_private_key_to_filename(crypto_pk_t *env, - const char *fname); - -int crypto_pk_check_key(crypto_pk_t *env); -int crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b); -int crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b); -size_t crypto_pk_keysize(const crypto_pk_t *env); -int crypto_pk_num_bits(crypto_pk_t *env); -crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig); -crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig); -int crypto_pk_key_is_private(const crypto_pk_t *key); -int crypto_pk_public_exponent_ok(crypto_pk_t *env); - -int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen, int padding); -int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen, - int padding, int warnOnFailure); -int crypto_pk_public_checksig(const crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen); -int crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, size_t siglen); -int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen); -int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen); -int crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen, - int padding, int force); -int crypto_pk_private_hybrid_decrypt(crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen, - int padding, int warnOnFailure); - -int crypto_pk_asn1_encode(crypto_pk_t *pk, char *dest, size_t dest_len); -crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len); -int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out); -int crypto_pk_get_common_digests(crypto_pk_t *pk, - common_digests_t *digests_out); -int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space); -int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out); - -int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out); -crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len); +void crypto_cipher_free_(crypto_cipher_t *env); +#define crypto_cipher_free(c) \ + FREE_AND_NULL(crypto_cipher_t, crypto_cipher_free_, (c)) /* symmetric crypto */ const char *crypto_cipher_get_key(crypto_cipher_t *env); @@ -222,44 +78,6 @@ int crypto_cipher_decrypt_with_iv(const char *key, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 and other digests. */ -int crypto_digest(char *digest, const char *m, size_t len); -int crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm); -int crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm); -int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len); -struct smartlist_t; -void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const struct smartlist_t *lst, - const char *append, - digest_algorithm_t alg); -void crypto_digest_smartlist(char *digest_out, size_t len_out, - const struct smartlist_t *lst, const char *append, - digest_algorithm_t alg); -const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); -size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); -int crypto_digest_algorithm_parse_name(const char *name); -crypto_digest_t *crypto_digest_new(void); -crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); -crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm); -void crypto_digest_free(crypto_digest_t *digest); -void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len); -void crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len); -crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest); -void crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from); -void crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len); -crypto_xof_t *crypto_xof_new(void); -void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); -void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); -void crypto_xof_free(crypto_xof_t *xof); - /* Key negotiation */ #define DH_TYPE_CIRCUIT 1 #define DH_TYPE_REND 2 @@ -273,7 +91,8 @@ int crypto_dh_get_public(crypto_dh_t *dh, char *pubkey_out, ssize_t crypto_dh_compute_secret(int severity, crypto_dh_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_out_len); -void crypto_dh_free(crypto_dh_t *dh); +void crypto_dh_free_(crypto_dh_t *dh); +#define crypto_dh_free(dh) FREE_AND_NULL(crypto_dh_t, crypto_dh_free_, (dh)) int crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len, @@ -284,57 +103,12 @@ int crypto_expand_key_material_rfc5869_sha256( const uint8_t *info_in, size_t info_in_len, uint8_t *key_out, size_t key_out_len); -/* random numbers */ -int crypto_seed_rng(void) ATTR_WUR; -MOCK_DECL(void,crypto_rand,(char *to, size_t n)); -void crypto_rand_unmocked(char *to, size_t n); -void crypto_strongest_rand(uint8_t *out, size_t out_len); -int crypto_rand_int(unsigned int max); -int crypto_rand_int_range(unsigned int min, unsigned int max); -uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); -time_t crypto_rand_time_range(time_t min, time_t max); -uint64_t crypto_rand_uint64(uint64_t max); -double crypto_rand_double(void); -struct tor_weak_rng_t; -void crypto_seed_weak_rng(struct tor_weak_rng_t *rng); -int crypto_init_siphash_key(void); - -char *crypto_random_hostname(int min_rand_len, int max_rand_len, - const char *prefix, const char *suffix); - -struct smartlist_t; -void *smartlist_choose(const struct smartlist_t *sl); -void smartlist_shuffle(struct smartlist_t *sl); - -/** OpenSSL-based utility functions. */ -void memwipe(void *mem, uint8_t byte, size_t sz); - /* Prototypes for private functions only used by tortls.c, crypto.c, and the * unit tests. */ -struct rsa_st; -struct evp_pkey_st; struct dh_st; -struct rsa_st *crypto_pk_get_rsa_(crypto_pk_t *env); -crypto_pk_t *crypto_new_pk_from_rsa_(struct rsa_st *rsa); -MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_evp_pkey_,(crypto_pk_t *env, - int private)); struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh); void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in); -#ifdef CRYPTO_PRIVATE -STATIC int crypto_force_rand_ssleay(void); -STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); - -#ifdef TOR_UNIT_TESTS -extern int break_strongest_rng_syscall; -extern int break_strongest_rng_fallback; -#endif -#endif - -#ifdef TOR_UNIT_TESTS -void crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src); -#endif - -#endif +#endif /* !defined(TOR_CRYPTO_H) */ diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index fcbee3aba2..996d94c6e2 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -21,9 +21,11 @@ #include <sys/stat.h> #endif #include "container.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_format.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "util.h" #include "torlog.h" @@ -43,7 +45,7 @@ int curve25519_donna(uint8_t *mypublic, #elif defined(HAVE_NACL_CRYPTO_SCALARMULT_CURVE25519_H) #include <nacl/crypto_scalarmult_curve25519.h> #endif -#endif +#endif /* defined(USE_CURVE25519_NACL) */ static void pick_curve25519_basepoint_impl(void); @@ -72,7 +74,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret, r = crypto_scalarmult_curve25519(output, secret, bp); #else #error "No implementation of curve25519 is available." -#endif +#endif /* defined(USE_CURVE25519_DONNA) || ... */ memwipe(bp, 0, sizeof(bp)); return r; } @@ -80,7 +82,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret, /** * Helper function: Multiply the scalar "secret" by the Curve25519 * basepoint (X=9), and store the result in "output". Return 0 on - * success, -1 on false. + * success, -1 on failure. */ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret) @@ -318,8 +320,11 @@ curve25519_basepoint_spot_check(void) } goto end; + // LCOV_EXCL_START -- we can only hit this code if there is a bug in our + // curve25519-basepoint implementation. fail: r = -1; + // LCOV_EXCL_STOP end: curve25519_use_ed = save_use_ed; return r; diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 4011820949..4834fa0836 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_CURVE25519_H @@ -6,6 +6,8 @@ #include "testsupport.h" #include "torint.h" +#include "crypto_digest.h" +#include "crypto_openssl_mgt.h" /** Length of a curve25519 public key when encoded. */ #define CURVE25519_PUBKEY_LEN 32 @@ -71,7 +73,7 @@ STATIC int curve25519_impl(uint8_t *output, const uint8_t *secret, const uint8_t *basepoint); STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret); -#endif +#endif /* defined(CRYPTO_CURVE25519_PRIVATE) */ #define CURVE25519_BASE64_PADDED_LEN 44 @@ -83,5 +85,5 @@ int curve25519_public_to_base64(char *output, void curve25519_set_impl_params(int use_ed); void curve25519_init(void); -#endif +#endif /* !defined(TOR_CRYPTO_CURVE25519_H) */ diff --git a/src/common/crypto_digest.c b/src/common/crypto_digest.c new file mode 100644 index 0000000000..9f9a1a1e2c --- /dev/null +++ b/src/common/crypto_digest.c @@ -0,0 +1,583 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest.c + * \brief Block of functions related with digest and xof utilities and + * operations. + **/ + +#include "container.h" +#include "crypto_digest.h" +#include "crypto_openssl_mgt.h" +#include "crypto_util.h" +#include "torlog.h" + +#include "keccak-tiny/keccak-tiny.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/hmac.h> +#include <openssl/sha.h> + +ENABLE_GCC_WARNING(redundant-decls) + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in + * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. + * Return 0 on success, -1 on failure. + */ +int +crypto_digest(char *digest, const char *m, size_t len) +{ + tor_assert(m); + tor_assert(digest); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) + return -1; + return 0; +} + +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); + else + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); + else + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + + if (!ret) + return -1; + return 0; +} + +/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the + * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len) +{ + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0) + return -1; + + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + case DIGEST_SHA512: + return "sha512"; + case DIGEST_SHA3_256: + return "sha3-256"; + case DIGEST_SHA3_512: + return "sha3-512"; + // LCOV_EXCL_START + default: + tor_fragile_assert(); + return "??unknown_digest??"; + // LCOV_EXCL_STOP + } +} + +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else if (!strcmp(name, "sha512")) + return DIGEST_SHA512; + else if (!strcmp(name, "sha3-256")) + return DIGEST_SHA3_256; + else if (!strcmp(name, "sha3-512")) + return DIGEST_SHA3_512; + else + return -1; +} + +/** Given an algorithm, return the digest length in bytes. */ +size_t +crypto_digest_algorithm_get_length(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return DIGEST_LEN; + case DIGEST_SHA256: + return DIGEST256_LEN; + case DIGEST_SHA512: + return DIGEST512_LEN; + case DIGEST_SHA3_256: + return DIGEST256_LEN; + case DIGEST_SHA3_512: + return DIGEST512_LEN; + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; /* Unreachable */ // LCOV_EXCL_LINE + } +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. Note also + * that space for other members might not even be allocated! + */ + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + SHA512_CTX sha512; /**< state for SHA512 */ + keccak_state sha3; /**< state for SHA3-[256,512] */ + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: + return END_OF_FIELD(d.sha1); + case DIGEST_SHA256: + return END_OF_FIELD(d.sha2); + case DIGEST_SHA512: + return END_OF_FIELD(d.sha512); + case DIGEST_SHA3_256: + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using <b>algorithm</b>. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add <b>len</b> bytes from <b>data</b> to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + case DIGEST_SHA512: + SHA512_Update(&digest->d.sha512, (void*)data, len); + break; + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to <b>out</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + crypto_digest_t tmpenv; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; + } + + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, alloc_bytes); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + SHA256_Final(r, &tmpenv.d.sha2); + break; + case DIGEST_SHA512: + SHA512_Final(r, &tmpenv.d.sha512); + break; +//LCOV_EXCL_START + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + default: + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); + break; +//LCOV_EXCL_STOP + } + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * <b>digest</b> + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + return tor_memdup(digest, alloc_bytes); +} + +/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of <b>digest</b> from <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object <b>into</b> with the state + * of the digest object <b>from</b>. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * plus the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of: the + * optional string <b>prepend</b>, those strings, + * and the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>len_out</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using + * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte + * result in <b>hmac_out</b>. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + unsigned char *rv = NULL; + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, + (unsigned char*)hmac_out, NULL); + tor_assert(rv); +} + +/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a + * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length + * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in + * <b>mac_out</b>. This function can't fail. */ +void +crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len) +{ + crypto_digest_t *digest; + + const uint64_t key_len_netorder = tor_htonll(key_len); + + tor_assert(mac_out); + tor_assert(key); + tor_assert(msg); + + digest = crypto_digest256_new(DIGEST_SHA3_256); + + /* Order matters here that is any subsystem using this function should + * expect this very precise ordering in the MAC construction. */ + crypto_digest_add_bytes(digest, (const char *) &key_len_netorder, + sizeof(key_len_netorder)); + crypto_digest_add_bytes(digest, (const char *) key, key_len); + crypto_digest_add_bytes(digest, (const char *) msg, msg_len); + crypto_digest_get_digest(digest, (char *) mac_out, len_out); + crypto_digest_free(digest); +} + +/* xof functions */ + +/** Internal state for a eXtendable-Output Function (XOF). */ +struct crypto_xof_t { + keccak_state s; +}; + +/** Allocate a new XOF object backed by SHAKE-256. The security level + * provided is a function of the length of the output used. Read and + * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output + * Functions" before using this construct. + */ +crypto_xof_t * +crypto_xof_new(void) +{ + crypto_xof_t *xof; + xof = tor_malloc(sizeof(crypto_xof_t)); + keccak_xof_init(&xof->s, 256); + return xof; +} + +/** Absorb bytes into a XOF object. Must not be called after a call to + * crypto_xof_squeeze_bytes() for the same instance, and will assert + * if attempted. + */ +void +crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) +{ + int i = keccak_xof_absorb(&xof->s, data, len); + tor_assert(i == 0); +} + +/** Squeeze bytes out of a XOF object. Calling this routine will render + * the XOF instance ineligible to absorb further data. + */ +void +crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) +{ + int i = keccak_xof_squeeze(&xof->s, out, len); + tor_assert(i == 0); +} + +/** Cleanse and deallocate a XOF object. */ +void +crypto_xof_free_(crypto_xof_t *xof) +{ + if (!xof) + return; + memwipe(xof, 0, sizeof(crypto_xof_t)); + tor_free(xof); +} + diff --git a/src/common/crypto_digest.h b/src/common/crypto_digest.h new file mode 100644 index 0000000000..3bd74acdfa --- /dev/null +++ b/src/common/crypto_digest.h @@ -0,0 +1,136 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest.h + * + * \brief Headers for crypto_digest.c + **/ + +#ifndef TOR_CRYPTO_DIGEST_H +#define TOR_CRYPTO_DIGEST_H + +#include <stdio.h> + +#include "container.h" +#include "torint.h" + +/** Length of the output of our message digest. */ +#define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but it could be any other 256-bit digest.) */ +#define DIGEST256_LEN 32 +/** Length of the output of our 64-bit optimized message digests (SHA512). */ +#define DIGEST512_LEN 64 + +/** Length of a sha1 message digest when encoded in base32 with trailing = + * signs removed. */ +#define BASE32_DIGEST_LEN 32 +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 +/** Length of a sha512 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST512_LEN 86 + +/** Length of hex encoding of SHA1 digest, not including final NUL. */ +#define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 +/** Length of hex encoding of SHA512 digest, not including final NUL. */ +#define HEX_DIGEST512_LEN 128 + +typedef enum { + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, + DIGEST_SHA512 = 2, + DIGEST_SHA3_256 = 3, + DIGEST_SHA3_512 = 4, +} digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1) +#define N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +#define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512) +/** Structure used to temporarily save the a digest object. Only implemented + * for SHA1 digest for now. */ +typedef struct crypto_digest_checkpoint_t { + uint8_t mem[DIGEST_CHECKPOINT_BYTES]; +} crypto_digest_checkpoint_t; + +/** A set of all the digests we commonly compute, taken on a single + * string. Any digests that are shorter than 512 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 44 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} common_digests_t; + +typedef struct crypto_digest_t crypto_digest_t; +typedef struct crypto_xof_t crypto_xof_t; + +/* SHA-1 and other digests */ +int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len); +void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const struct smartlist_t *lst, + const char *append, + digest_algorithm_t alg); +void crypto_digest_smartlist(char *digest_out, size_t len_out, + const struct smartlist_t *lst, const char *append, + digest_algorithm_t alg); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); +crypto_digest_t *crypto_digest_new(void); +crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); +crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm); +void crypto_digest_free_(crypto_digest_t *digest); +#define crypto_digest_free(d) \ + FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d)) +void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len); +void crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len); +crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest); +void crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest); +void crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint); +void crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from); +void crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len); +void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len); + +/* xof functions*/ +crypto_xof_t *crypto_xof_new(void); +void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); +void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); +void crypto_xof_free_(crypto_xof_t *xof); +#define crypto_xof_free(xof) \ + FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) + +#ifdef TOR_UNIT_TESTS +digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); +#endif + +#endif /* !defined(TOR_CRYPTO_DIGEST_H) */ + diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index 30ed772274..9c13e3bdf0 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,26 +15,26 @@ * keys to and from the corresponding Curve25519 keys. */ +#define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif -#include "crypto.h" - #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "torlog.h" #include "util.h" +#include "util_format.h" #include "ed25519/ref10/ed25519_ref10.h" #include "ed25519/donna/ed25519_donna_tor.h" -#include <openssl/sha.h> - static void pick_ed25519_impl(void); -static int ed25519_impl_spot_check(void); /** An Ed25519 implementation, as a set of function pointers. */ typedef struct { @@ -59,6 +59,9 @@ typedef struct { int (*pubkey_from_curve25519_pubkey)(unsigned char *, const unsigned char *, int); + + int (*ed25519_scalarmult_with_group_order)(unsigned char *, + const unsigned char *); } ed25519_impl_t; /** The Ref10 Ed25519 implementation. This one is pure C and lightly @@ -79,6 +82,7 @@ static const ed25519_impl_t impl_ref10 = { ed25519_ref10_blind_public_key, ed25519_ref10_pubkey_from_curve25519_pubkey, + ed25519_ref10_scalarmult_with_group_order, }; /** The Ref10 Ed25519 implementation. This one is heavily optimized, but still @@ -99,6 +103,7 @@ static const ed25519_impl_t impl_donna = { ed25519_donna_blind_public_key, ed25519_donna_pubkey_from_curve25519_pubkey, + ed25519_donna_scalarmult_with_group_order, }; /** Which Ed25519 implementation are we using? NULL if we haven't decided @@ -147,7 +152,7 @@ crypto_ed25519_testing_restore_impl(void) ed25519_impl = saved_ed25519_impl; saved_ed25519_impl = NULL; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** * Initialize a new ed25519 secret key in <b>seckey_out</b>. If @@ -211,10 +216,18 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong) return 0; } +/** Return true iff 'pubkey' is set to zero (eg to indicate that it is not + * set). */ +int +ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey) +{ + return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); +} + /* Return a heap-allocated array that contains <b>msg</b> prefixed by the * string <b>prefix_str</b>. Set <b>final_msg_len_out</b> to the size of the - * final array. If an error occured, return NULL. It's the resonsibility of the - * caller to free the returned array. */ + * final array. If an error occurred, return NULL. It's the responsibility of + * the caller to free the returned array. */ static uint8_t * get_prefixed_msg(const uint8_t *msg, size_t msg_len, const char *prefix_str, @@ -247,7 +260,7 @@ get_prefixed_msg(const uint8_t *msg, size_t msg_len, * Set <b>signature_out</b> to a signature of the <b>len</b>-byte message * <b>msg</b>, using the secret and public key in <b>keypair</b>. * - * Return 0 if we successfuly signed the message, otherwise return -1. + * Return 0 if we successfully signed the message, otherwise return -1. */ int ed25519_sign(ed25519_signature_t *signature_out, @@ -267,11 +280,11 @@ ed25519_sign(ed25519_signature_t *signature_out, * Like ed25519_sign(), but also prefix <b>msg</b> with <b>prefix_str</b> * before signing. <b>prefix_str</b> must be a NUL-terminated string. */ -int -ed25519_sign_prefixed(ed25519_signature_t *signature_out, - const uint8_t *msg, size_t msg_len, - const char *prefix_str, - const ed25519_keypair_t *keypair) +MOCK_IMPL(int, +ed25519_sign_prefixed,(ed25519_signature_t *signature_out, + const uint8_t *msg, size_t msg_len, + const char *prefix_str, + const ed25519_keypair_t *keypair)) { int retval; size_t prefixed_msg_len; @@ -281,9 +294,12 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out, prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str, &prefixed_msg_len); - if (!prefixed_msg) { + if (BUG(!prefixed_msg)) { + /* LCOV_EXCL_START -- only possible when the message and prefix are + * ridiculously huge */ log_warn(LD_GENERAL, "Failed to get prefixed msg."); return -1; + /* LCOV_EXCL_STOP */ } retval = ed25519_sign(signature_out, @@ -300,10 +316,10 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out, * * Return 0 if the signature is valid; -1 if it isn't. */ -int -ed25519_checksig(const ed25519_signature_t *signature, - const uint8_t *msg, size_t len, - const ed25519_public_key_t *pubkey) +MOCK_IMPL(int, +ed25519_checksig,(const ed25519_signature_t *signature, + const uint8_t *msg, size_t len, + const ed25519_public_key_t *pubkey)) { return get_ed_impl()->open(signature->sig, msg, len, pubkey->pubkey) < 0 ? -1 : 0; @@ -326,9 +342,12 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature, prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str, &prefixed_msg_len); - if (!prefixed_msg) { + if (BUG(!prefixed_msg)) { + /* LCOV_EXCL_START -- only possible when the message and prefix are + * ridiculously huge */ log_warn(LD_GENERAL, "Failed to get prefixed msg."); return -1; + /* LCOV_EXCL_STOP */ } retval = ed25519_checksig(signature, @@ -346,10 +365,10 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature, * was valid. Otherwise return -N, where N is the number of invalid * signatures. */ -int -ed25519_checksig_batch(int *okay_out, - const ed25519_checkable_t *checkable, - int n_checkable) +MOCK_IMPL(int, +ed25519_checksig_batch,(int *okay_out, + const ed25519_checkable_t *checkable, + int n_checkable)) { int i, res; const ed25519_impl_t *impl = get_ed_impl(); @@ -434,14 +453,16 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, { const char string[] = "Derive high part of ed25519 key from curve25519 key"; ed25519_public_key_t pubkey_check; - SHA512_CTX ctx; - uint8_t sha512_output[64]; + crypto_digest_t *ctx; + uint8_t sha512_output[DIGEST512_LEN]; memcpy(out->seckey.seckey, inp->seckey.secret_key, 32); - SHA512_Init(&ctx); - SHA512_Update(&ctx, out->seckey.seckey, 32); - SHA512_Update(&ctx, string, sizeof(string)); - SHA512_Final(sha512_output, &ctx); + + ctx = crypto_digest512_new(DIGEST_SHA512); + crypto_digest_add_bytes(ctx, (const char*)out->seckey.seckey, 32); + crypto_digest_add_bytes(ctx, (const char*)string, sizeof(string)); + crypto_digest_get_digest(ctx, (char *)sha512_output, sizeof(sha512_output)); + crypto_digest_free(ctx); memcpy(out->seckey.seckey + 32, sha512_output, 32); ed25519_public_key_generate(&out->pubkey, &out->seckey); @@ -454,7 +475,6 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32)); memwipe(&pubkey_check, 0, sizeof(pubkey_check)); - memwipe(&ctx, 0, sizeof(ctx)); memwipe(sha512_output, 0, sizeof(sha512_output)); return 0; @@ -483,7 +503,8 @@ ed25519_public_key_from_curve25519_public_key(ed25519_public_key_t *pubkey, * service descriptors are encrypted with a key derived from the service's * long-term public key, and then signed with (and stored at a position * indexed by) a short-term key derived by blinding the long-term keys. - */ + * + * Return 0 if blinding was successful, else return -1. */ int ed25519_keypair_blind(ed25519_keypair_t *out, const ed25519_keypair_t *inp, @@ -494,7 +515,9 @@ ed25519_keypair_blind(ed25519_keypair_t *out, get_ed_impl()->blind_secret_key(out->seckey.seckey, inp->seckey.seckey, param); - ed25519_public_blind(&pubkey_check, &inp->pubkey, param); + if (ed25519_public_blind(&pubkey_check, &inp->pubkey, param) < 0) { + return -1; + } ed25519_public_key_generate(&out->pubkey, &out->seckey); tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32)); @@ -507,15 +530,14 @@ ed25519_keypair_blind(ed25519_keypair_t *out, /** * Given an ed25519 public key in <b>inp</b>, generate a corresponding blinded * public key in <b>out</b>, blinded with the 32-byte parameter in - * <b>param</b>. Return 0 on sucess, -1 on railure. + * <b>param</b>. Return 0 on success, -1 on railure. */ int ed25519_public_blind(ed25519_public_key_t *out, const ed25519_public_key_t *inp, const uint8_t *param) { - get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param); - return 0; + return get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param); } /** @@ -601,7 +623,7 @@ ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out, /** Release all storage held for <b>kp</b>. */ void -ed25519_keypair_free(ed25519_keypair_t *kp) +ed25519_keypair_free_(ed25519_keypair_t *kp) { if (! kp) return; @@ -620,10 +642,22 @@ ed25519_pubkey_eq(const ed25519_public_key_t *key1, return tor_memeq(key1->pubkey, key2->pubkey, ED25519_PUBKEY_LEN); } +/** + * Set <b>dest</b> to contain the same key as <b>src</b>. + */ +void +ed25519_pubkey_copy(ed25519_public_key_t *dest, + const ed25519_public_key_t *src) +{ + tor_assert(dest); + tor_assert(src); + memcpy(dest, src, sizeof(ed25519_public_key_t)); +} + /** Check whether the given Ed25519 implementation seems to be working. * If so, return 0; otherwise return -1. */ -static int -ed25519_impl_spot_check(void) +MOCK_IMPL(STATIC int, +ed25519_impl_spot_check,(void)) { static const uint8_t alicesk[32] = { 0xc5,0xaa,0x8d,0xf4,0x3f,0x9f,0x83,0x7b, @@ -686,13 +720,16 @@ ed25519_impl_spot_check(void) return -1; /* XXX/yawning: Someone that's more paranoid than I am, can write "Assume - * ref0 is cannonical, and fuzz impl against it" if they want, but I doubt + * ref0 is canonical, and fuzz impl against it" if they want, but I doubt * that will catch anything that the known answer tests won't. */ goto end; + // LCOV_EXCL_START -- We can only reach this if our ed25519 implementation is + // broken. fail: r = -1; + // LCOV_EXCL_STOP end: return r; } @@ -726,7 +763,7 @@ pick_ed25519_impl(void) /* LCOV_EXCL_STOP */ } -/* Initialize the Ed25519 implementation. This is neccessary if you're +/* Initialize the Ed25519 implementation. This is necessary if you're * going to use them in a multithreaded setting, and not otherwise. */ void ed25519_init(void) @@ -734,3 +771,47 @@ ed25519_init(void) pick_ed25519_impl(); } +/* Return true if <b>point</b> is the identity element of the ed25519 group. */ +static int +ed25519_point_is_identity_element(const uint8_t *point) +{ + /* The identity element in ed25159 is the point with coordinates (0,1). */ + static const uint8_t ed25519_identity[32] = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + tor_assert(sizeof(ed25519_identity) == ED25519_PUBKEY_LEN); + return tor_memeq(point, ed25519_identity, sizeof(ed25519_identity)); +} + +/** Validate <b>pubkey</b> to ensure that it has no torsion component. + * Return 0 if <b>pubkey</b> is valid, else return -1. */ +int +ed25519_validate_pubkey(const ed25519_public_key_t *pubkey) +{ + uint8_t result[32] = {9}; + + /* First check that we were not given the identity element */ + if (ed25519_point_is_identity_element(pubkey->pubkey)) { + log_warn(LD_CRYPTO, "ed25519 pubkey is the identity"); + return -1; + } + + /* For any point on the curve, doing l*point should give the identity element + * (where l is the group order). Do the computation and check that the + * identity element is returned. */ + if (get_ed_impl()->ed25519_scalarmult_with_group_order(result, + pubkey->pubkey) < 0) { + log_warn(LD_CRYPTO, "ed25519 group order scalarmult failed"); + return -1; + } + + if (!ed25519_point_is_identity_element(result)) { + log_warn(LD_CRYPTO, "ed25519 validation failed"); + return -1; + } + + return 0; +} + diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h index 31afc49ccc..74269ccffd 100644 --- a/src/common/crypto_ed25519.h +++ b/src/common/crypto_ed25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_ED25519_H @@ -7,6 +7,7 @@ #include "testsupport.h" #include "torint.h" #include "crypto_curve25519.h" +#include "util.h" #define ED25519_PUBKEY_LEN 32 #define ED25519_SECKEY_LEN 64 @@ -51,21 +52,24 @@ int ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong); int ed25519_sign(ed25519_signature_t *signature_out, const uint8_t *msg, size_t len, const ed25519_keypair_t *key); -int ed25519_checksig(const ed25519_signature_t *signature, - const uint8_t *msg, size_t len, - const ed25519_public_key_t *pubkey); +MOCK_DECL(int,ed25519_checksig,(const ed25519_signature_t *signature, + const uint8_t *msg, size_t len, + const ed25519_public_key_t *pubkey)); + +MOCK_DECL(int, +ed25519_sign_prefixed,(ed25519_signature_t *signature_out, + const uint8_t *msg, size_t len, + const char *prefix_str, + const ed25519_keypair_t *keypair)); -int -ed25519_sign_prefixed(ed25519_signature_t *signature_out, - const uint8_t *msg, size_t len, - const char *prefix_str, - const ed25519_keypair_t *keypair); int ed25519_checksig_prefixed(const ed25519_signature_t *signature, const uint8_t *msg, size_t len, const char *prefix_str, const ed25519_public_key_t *pubkey); +int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey); + /** * A collection of information necessary to check an Ed25519 signature. Used * for batch verification. @@ -81,9 +85,9 @@ typedef struct { size_t len; } ed25519_checkable_t; -int ed25519_checksig_batch(int *okay_out, - const ed25519_checkable_t *checkable, - int n_checkable); +MOCK_DECL(int, ed25519_checksig_batch,(int *okay_out, + const ed25519_checkable_t *checkable, + int n_checkable)); int ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, int *signbit_out, @@ -114,18 +118,28 @@ int ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out, char **tag_out, const char *filename); -void ed25519_keypair_free(ed25519_keypair_t *kp); +void ed25519_keypair_free_(ed25519_keypair_t *kp); +#define ed25519_keypair_free(kp) \ + FREE_AND_NULL(ed25519_keypair_t, ed25519_keypair_free_, (kp)) int ed25519_pubkey_eq(const ed25519_public_key_t *key1, const ed25519_public_key_t *key2); +void ed25519_pubkey_copy(ed25519_public_key_t *dest, + const ed25519_public_key_t *src); void ed25519_set_impl_params(int use_donna); void ed25519_init(void); +int ed25519_validate_pubkey(const ed25519_public_key_t *pubkey); + #ifdef TOR_UNIT_TESTS void crypto_ed25519_testing_force_impl(const char *name); void crypto_ed25519_testing_restore_impl(void); #endif +#ifdef CRYPTO_ED25519_PRIVATE +MOCK_DECL(STATIC int, ed25519_impl_spot_check, (void)); #endif +#endif /* !defined(TOR_CRYPTO_ED25519_H) */ + diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index 2f6d847c83..460e85bac1 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,10 +15,11 @@ #include <sys/stat.h> #endif #include "container.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_util.h" #include "util.h" #include "util_format.h" #include "torlog.h" @@ -161,6 +162,27 @@ curve25519_public_from_base64(curve25519_public_key_t *pkey, } } +/** For logging convenience: Convert <b>pkey</b> to a statically allocated + * base64 string and return it. Not threadsafe. Format not meant to be + * computer-readable; it may change in the future. Subsequent calls invalidate + * previous returns. */ +const char * +ed25519_fmt(const ed25519_public_key_t *pkey) +{ + static char formatted[ED25519_BASE64_LEN+1]; + if (pkey) { + if (ed25519_public_key_is_zero(pkey)) { + strlcpy(formatted, "<unset>", sizeof(formatted)); + } else { + int r = ed25519_public_to_base64(formatted, pkey); + tor_assert(!r); + } + } else { + strlcpy(formatted, "<null>", sizeof(formatted)); + } + return formatted; +} + /** Try to decode the string <b>input</b> into an ed25519 public key. On * success, store the value in <b>pkey</b> and return 0. Otherwise return * -1. */ diff --git a/src/common/crypto_format.h b/src/common/crypto_format.h index 012e228cc4..bbd85dc720 100644 --- a/src/common/crypto_format.h +++ b/src/common/crypto_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_FORMAT_H @@ -28,6 +28,7 @@ int ed25519_public_from_base64(ed25519_public_key_t *pkey, const char *input); int ed25519_public_to_base64(char *output, const ed25519_public_key_t *pkey); +const char *ed25519_fmt(const ed25519_public_key_t *pkey); /* XXXX move these to crypto_format.h */ #define ED25519_SIG_BASE64_LEN 86 @@ -42,5 +43,5 @@ int digest_from_base64(char *digest, const char *d64); int digest256_to_base64(char *d64, const char *digest); int digest256_from_base64(char *digest, const char *d64); -#endif +#endif /* !defined(TOR_CRYPTO_FORMAT_H) */ diff --git a/src/common/crypto_openssl_mgt.c b/src/common/crypto_openssl_mgt.c new file mode 100644 index 0000000000..ea3519efa2 --- /dev/null +++ b/src/common/crypto_openssl_mgt.c @@ -0,0 +1,161 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_openssl.c + * + * \brief Block of functions related to operations from OpenSSL. + **/ + +#include "compat_openssl.h" +#include "crypto_openssl_mgt.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/err.h> +#include <openssl/rsa.h> +#include <openssl/pem.h> +#include <openssl/evp.h> +#include <openssl/engine.h> +#include <openssl/rand.h> +#include <openssl/bn.h> +#include <openssl/dh.h> +#include <openssl/conf.h> +#include <openssl/hmac.h> +#include <openssl/crypto.h> + +ENABLE_GCC_WARNING(redundant-decls) + +#ifndef NEW_THREAD_API +/** A number of preallocated mutexes for use by OpenSSL. */ +static tor_mutex_t **openssl_mutexes_ = NULL; +/** How many mutexes have we allocated for use by OpenSSL? */ +static int n_openssl_mutexes_ = 0; +#endif /* !defined(NEW_THREAD_API) */ + +/** Declare STATIC functions */ +STATIC char * parse_openssl_version_str(const char *raw_version); +#ifndef NEW_THREAD_API +STATIC void openssl_locking_cb_(int mode, int n, const char *file, int line); +STATIC void tor_set_openssl_thread_id(CRYPTO_THREADID *threadid); +#endif + +/* Returns a trimmed and human-readable version of an openssl version string +* <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10 +* May 2012' and this will parse them into a form similar to '1.0.0b' */ +STATIC char * +parse_openssl_version_str(const char *raw_version) +{ + const char *end_of_version = NULL; + /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's + trim that down. */ + if (!strcmpstart(raw_version, "OpenSSL ")) { + raw_version += strlen("OpenSSL "); + end_of_version = strchr(raw_version, ' '); + } + + if (end_of_version) + return tor_strndup(raw_version, + end_of_version-raw_version); + else + return tor_strdup(raw_version); +} + +static char *crypto_openssl_version_str = NULL; +/* Return a human-readable version of the run-time openssl version number. */ +const char * +crypto_openssl_get_version_str(void) +{ + if (crypto_openssl_version_str == NULL) { + const char *raw_version = OpenSSL_version(OPENSSL_VERSION); + crypto_openssl_version_str = parse_openssl_version_str(raw_version); + } + return crypto_openssl_version_str; +} + +static char *crypto_openssl_header_version_str = NULL; +/* Return a human-readable version of the compile-time openssl version +* number. */ +const char * +crypto_openssl_get_header_version_str(void) +{ + if (crypto_openssl_header_version_str == NULL) { + crypto_openssl_header_version_str = + parse_openssl_version_str(OPENSSL_VERSION_TEXT); + } + return crypto_openssl_header_version_str; +} + +#ifndef OPENSSL_THREADS +#error OpenSSL has been built without thread support. Tor requires an \ + OpenSSL library with thread support enabled. +#endif + +#ifndef NEW_THREAD_API +/** Helper: OpenSSL uses this callback to manipulate mutexes. */ +STATIC void +openssl_locking_cb_(int mode, int n, const char *file, int line) +{ + (void)file; + (void)line; + if (!openssl_mutexes_) + /* This is not a really good fix for the + * "release-freed-lock-from-separate-thread-on-shutdown" problem, but + * it can't hurt. */ + return; + if (mode & CRYPTO_LOCK) + tor_mutex_acquire(openssl_mutexes_[n]); + else + tor_mutex_release(openssl_mutexes_[n]); +} + +STATIC void +tor_set_openssl_thread_id(CRYPTO_THREADID *threadid) +{ + CRYPTO_THREADID_set_numeric(threadid, tor_get_thread_id()); +} +#endif /* !defined(NEW_THREAD_API) */ + +/** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being + * multithreaded. Returns 0. */ +int +setup_openssl_threading(void) +{ +#ifndef NEW_THREAD_API + int i; + int n = CRYPTO_num_locks(); + n_openssl_mutexes_ = n; + openssl_mutexes_ = tor_calloc(n, sizeof(tor_mutex_t *)); + for (i=0; i < n; ++i) + openssl_mutexes_[i] = tor_mutex_new(); + CRYPTO_set_locking_callback(openssl_locking_cb_); + CRYPTO_THREADID_set_callback(tor_set_openssl_thread_id); +#endif /* !defined(NEW_THREAD_API) */ + return 0; +} + +/** free OpenSSL variables */ +void +crypto_openssl_free_all(void) +{ + tor_free(crypto_openssl_version_str); + tor_free(crypto_openssl_header_version_str); + +#ifndef NEW_THREAD_API + if (n_openssl_mutexes_) { + int n = n_openssl_mutexes_; + tor_mutex_t **ms = openssl_mutexes_; + int i; + openssl_mutexes_ = NULL; + n_openssl_mutexes_ = 0; + for (i=0;i<n;++i) { + tor_mutex_free(ms[i]); + } + tor_free(ms); + } +#endif /* !defined(NEW_THREAD_API) */ +} + diff --git a/src/common/crypto_openssl_mgt.h b/src/common/crypto_openssl_mgt.h new file mode 100644 index 0000000000..09b6737962 --- /dev/null +++ b/src/common/crypto_openssl_mgt.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_openssl.h + * + * \brief Headers for crypto_openssl.c + **/ + +#ifndef TOR_CRYPTO_OPENSSL_H +#define TOR_CRYPTO_OPENSSL_H + +#include <stdio.h> +#include "util.h" + +#include <openssl/engine.h> + +/* + Macro to create an arbitrary OpenSSL version number as used by + OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard + to read. + + Don't use this directly, instead use one of the other OPENSSL_V macros + below. + + The format is: 4 bits major, 8 bits minor, 8 bits fix, 8 bits patch, 4 bit + status. + */ +#define OPENSSL_VER(a,b,c,d,e) \ + (((a)<<28) | \ + ((b)<<20) | \ + ((c)<<12) | \ + ((d)<< 4) | \ + (e)) +/** An openssl release number. For example, OPENSSL_V(0,9,8,'j') is the + * version for the released version of 0.9.8j */ +#define OPENSSL_V(a,b,c,d) \ + OPENSSL_VER((a),(b),(c),(d)-'a'+1,0xf) +/** An openssl release number for the first release in the series. For + * example, OPENSSL_V_NOPATCH(1,0,0) is the first released version of OpenSSL + * 1.0.0. */ +#define OPENSSL_V_NOPATCH(a,b,c) \ + OPENSSL_VER((a),(b),(c),0,0xf) +/** The first version that would occur for any alpha or beta in an openssl + * series. For example, OPENSSL_V_SERIES(0,9,8) is greater than any released + * 0.9.7, and less than any released 0.9.8. */ +#define OPENSSL_V_SERIES(a,b,c) \ + OPENSSL_VER((a),(b),(c),0,0) + +#ifdef ANDROID +/* Android's OpenSSL seems to have removed all of its Engine support. */ +#define DISABLE_ENGINES +#endif + +#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && \ + !defined(LIBRESSL_VERSION_NUMBER) +/* OpenSSL as of 1.1.0pre4 has an "new" thread API, which doesn't require + * seting up various callbacks. + * + * OpenSSL 1.1.0pre4 has a messed up `ERR_remove_thread_state()` prototype, + * while the previous one was restored in pre5, and the function made a no-op + * (along with a deprecated annotation, which produces a compiler warning). + * + * While it is possible to support all three versions of the thread API, + * a version that existed only for one snapshot pre-release is kind of + * pointless, so let's not. + */ +#define NEW_THREAD_API +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && ... */ + +/* global openssl state */ +const char * crypto_openssl_get_version_str(void); +const char * crypto_openssl_get_header_version_str(void); + +/* OpenSSL threading setup function */ +int setup_openssl_threading(void); + +/* Tor OpenSSL utility functions */ +void crypto_openssl_free_all(void); + +#endif /* !defined(TOR_CRYPTO_OPENSSL_H) */ + diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 31e37c007d..c2bd1d26cb 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,8 +9,11 @@ */ #include "crypto.h" -#include "crypto_s2k.h" +#include "crypto_digest.h" #include "crypto_pwbox.h" +#include "crypto_rand.h" +#include "crypto_s2k.h" +#include "crypto_util.h" #include "di_ops.h" #include "util.h" #include "pwbox.h" @@ -107,7 +110,6 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, rv = 0; goto out; - err: /* LCOV_EXCL_START This error case is often unreachable if we're correctly coded, unless @@ -123,6 +125,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, - pwbox_encoded_encode can't fail unless we're using trunnel wrong, or it's buggy. */ + err: tor_free(result); rv = -1; /* LCOV_EXCL_STOP */ diff --git a/src/common/crypto_pwbox.h b/src/common/crypto_pwbox.h index aadd477078..a26b6d2c17 100644 --- a/src/common/crypto_pwbox.h +++ b/src/common/crypto_pwbox.h @@ -1,3 +1,6 @@ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + #ifndef CRYPTO_PWBOX_H_INCLUDED_ #define CRYPTO_PWBOX_H_INCLUDED_ @@ -16,5 +19,5 @@ int crypto_unpwbox(uint8_t **out, size_t *outlen_out, const uint8_t *inp, size_t input_len, const char *secret, size_t secret_len); -#endif +#endif /* !defined(CRYPTO_PWBOX_H_INCLUDED_) */ diff --git a/src/common/crypto_rand.c b/src/common/crypto_rand.c new file mode 100644 index 0000000000..df2e2f65d3 --- /dev/null +++ b/src/common/crypto_rand.c @@ -0,0 +1,615 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand.c + * + * \brief Functions for initialising and seeding (pseudo-)random + * number generators, and working with randomness. + **/ + +#ifndef CRYPTO_RAND_PRIVATE +#define CRYPTO_RAND_PRIVATE + +#include "crypto_rand.h" + +#ifdef _WIN32 +#include <windows.h> +#include <wincrypt.h> +#endif /* defined(_WIN32) */ + +#include "container.h" +#include "compat.h" +#include "compat_openssl.h" +#include "crypto_util.h" +#include "sandbox.h" +#include "testsupport.h" +#include "torlog.h" +#include "util.h" +#include "util_format.h" + +DISABLE_GCC_WARNING(redundant-decls) +#include <openssl/rand.h> +ENABLE_GCC_WARNING(redundant-decls) + +#if __GNUC__ && GCC_VERSION >= 402 +#if GCC_VERSION >= 406 +#pragma GCC diagnostic pop +#else +#pragma GCC diagnostic warning "-Wredundant-decls" +#endif +#endif /* __GNUC__ && GCC_VERSION >= 402 */ + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_SYS_FCNTL_H +#include <sys/fcntl.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_SYS_SYSCALL_H +#include <sys/syscall.h> +#endif +#ifdef HAVE_SYS_RANDOM_H +#include <sys/random.h> +#endif + +/** + * How many bytes of entropy we add at once. + * + * This is how much entropy OpenSSL likes to add right now, so maybe it will + * work for us too. + **/ +#define ADD_ENTROPY 32 + +/** + * Longest recognized DNS query. + **/ +#define MAX_DNS_LABEL_SIZE 63 + +/** + * Largest strong entropy request permitted. + **/ +#define MAX_STRONGEST_RAND_SIZE 256 + +/** + * Set the seed of the weak RNG to a random value. + **/ +void +crypto_seed_weak_rng(tor_weak_rng_t *rng) +{ + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(rng, seed); +} + +#ifdef TOR_UNIT_TESTS +int break_strongest_rng_syscall = 0; +int break_strongest_rng_fallback = 0; +#endif + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on + * failure. A maximum request size of 256 bytes is imposed. + **/ +static int +crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) +{ + tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); + + /* We only log at notice-level here because in the case that this function + * fails the crypto_strongest_rand_raw() caller will log with a warning-level + * message and let crypto_strongest_rand() error out and finally terminating + * Tor with an assertion error. + */ + +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_syscall) + return -1; +#endif + +#if defined(_WIN32) + static int provider_set = 0; + static HCRYPTPROV provider; + + if (!provider_set) { + if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT)) { + log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1]."); + return -1; + } + provider_set = 1; + } + if (!CryptGenRandom(provider, out_len, out)) { + log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI."); + return -1; + } + + return 0; +#elif defined(__linux__) && defined(SYS_getrandom) + static int getrandom_works = 1; /* Be optimistic about our chances... */ + + /* getrandom() isn't as straightforward as getentropy(), and has + * no glibc wrapper. + * + * As far as I can tell from getrandom(2) and the source code, the + * requests we issue will always succeed (though it will block on the + * call if /dev/urandom isn't seeded yet), since we are NOT specifying + * GRND_NONBLOCK and the request is <= 256 bytes. + * + * The manpage is unclear on what happens if a signal interrupts the call + * while the request is blocked due to lack of entropy.... + * + * We optimistically assume that getrandom() is available and functional + * because it is the way of the future, and 2 branch mispredicts pale in + * comparison to the overheads involved with failing to open + * /dev/srandom followed by opening and reading from /dev/urandom. + */ + if (PREDICT_LIKELY(getrandom_works)) { + long ret; + /* A flag of '0' here means to read from '/dev/urandom', and to + * block if insufficient entropy is available to service the + * request. + */ + const unsigned int flags = 0; + do { + ret = syscall(SYS_getrandom, out, out_len, flags); + } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); + + if (PREDICT_UNLIKELY(ret == -1)) { + /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ + tor_assert(errno != EAGAIN); + tor_assert(errno != EINTR); + + /* Useful log message for errno. */ + if (errno == ENOSYS) { + log_notice(LD_CRYPTO, "Can't get entropy from getrandom()." + " You are running a version of Tor built to support" + " getrandom(), but the kernel doesn't implement this" + " function--probably because it is too old?" + " Trying fallback method instead."); + } else { + log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s." + " Trying fallback method instead.", + strerror(errno)); + } + + getrandom_works = 0; /* Don't bother trying again. */ + return -1; + /* LCOV_EXCL_STOP */ + } + + tor_assert(ret == (long)out_len); + return 0; + } + + return -1; /* getrandom() previously failed unexpectedly. */ +#elif defined(HAVE_GETENTROPY) + /* getentropy() is what Linux's getrandom() wants to be when it grows up. + * the only gotcha is that requests are limited to 256 bytes. + */ + return getentropy(out, out_len); +#else + (void) out; +#endif /* defined(_WIN32) || ... */ + + /* This platform doesn't have a supported syscall based random. */ + return -1; +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * via the per-platform fallback mechanism, storing it into <b>out</b>. + * Return 0 on success, -1 on failure. A maximum request size of 256 bytes + * is imposed. + **/ +static int +crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) +{ +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_fallback) + return -1; +#endif + +#ifdef _WIN32 + /* Windows exclusively uses crypto_strongest_rand_syscall(). */ + (void)out; + (void)out_len; + return -1; +#else /* !(defined(_WIN32)) */ + static const char *filenames[] = { + "/dev/srandom", "/dev/urandom", "/dev/random", NULL + }; + int fd, i; + size_t n; + + for (i = 0; filenames[i]; ++i) { + log_debug(LD_FS, "Considering %s as entropy source", filenames[i]); + fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); + if (fd<0) continue; + log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); + n = read_all(fd, (char*)out, out_len, 0); + close(fd); + if (n != out_len) { + /* LCOV_EXCL_START + * We can't make /dev/foorandom actually fail. */ + log_notice(LD_CRYPTO, + "Error reading from entropy source %s (read only %lu bytes).", + filenames[i], + (unsigned long)n); + return -1; + /* LCOV_EXCL_STOP */ + } + + return 0; + } + + return -1; +#endif /* defined(_WIN32) */ +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum + * request size of 256 bytes is imposed. + **/ +STATIC int +crypto_strongest_rand_raw(uint8_t *out, size_t out_len) +{ + static const size_t sanity_min_size = 16; + static const int max_attempts = 3; + tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); + + /* For buffers >= 16 bytes (128 bits), we sanity check the output by + * zero filling the buffer and ensuring that it actually was at least + * partially modified. + * + * Checking that any individual byte is non-zero seems like it would + * fail too often (p = out_len * 1/256) for comfort, but this is an + * "adjust according to taste" sort of check. + */ + memwipe(out, 0, out_len); + for (int i = 0; i < max_attempts; i++) { + /* Try to use the syscall/OS favored mechanism to get strong entropy. */ + if (crypto_strongest_rand_syscall(out, out_len) != 0) { + /* Try to use the less-favored mechanism to get strong entropy. */ + if (crypto_strongest_rand_fallback(out, out_len) != 0) { + /* Welp, we tried. Hopefully the calling code terminates the process + * since we're basically boned without good entropy. + */ + log_warn(LD_CRYPTO, + "Cannot get strong entropy: no entropy source found."); + return -1; + } + } + + if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) + return 0; + } + + /* LCOV_EXCL_START + * + * We tried max_attempts times to fill a buffer >= 128 bits long, + * and each time it returned all '0's. Either the system entropy + * source is busted, or the user should go out and buy a ticket to + * every lottery on the planet. + */ + log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); + + return -1; + /* LCOV_EXCL_STOP */ +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * storing it into <b>out</b>. + **/ +void +crypto_strongest_rand(uint8_t *out, size_t out_len) +{ +#define DLEN SHA512_DIGEST_LENGTH + /* We're going to hash DLEN bytes from the system RNG together with some + * bytes from the openssl PRNG, in order to yield DLEN bytes. + */ + uint8_t inp[DLEN*2]; + uint8_t tmp[DLEN]; + tor_assert(out); + while (out_len) { + crypto_rand((char*) inp, DLEN); + if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { + // LCOV_EXCL_START + log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " + "important key. Exiting."); + /* Die with an assertion so we get a stack trace. */ + tor_assert(0); + // LCOV_EXCL_STOP + } + if (out_len >= DLEN) { + SHA512(inp, sizeof(inp), out); + out += DLEN; + out_len -= DLEN; + } else { + SHA512(inp, sizeof(inp), tmp); + memcpy(out, tmp, out_len); + break; + } + } + memwipe(tmp, 0, sizeof(tmp)); + memwipe(inp, 0, sizeof(inp)); +#undef DLEN +} + +/** + * Seed OpenSSL's random number generator with bytes from the operating + * system. Return 0 on success, -1 on failure. + **/ +int +crypto_seed_rng(void) +{ + int rand_poll_ok = 0, load_entropy_ok = 0; + uint8_t buf[ADD_ENTROPY]; + + /* OpenSSL has a RAND_poll function that knows about more kinds of + * entropy than we do. We'll try calling that, *and* calling our own entropy + * functions. If one succeeds, we'll accept the RNG as seeded. */ + rand_poll_ok = RAND_poll(); + if (rand_poll_ok == 0) + log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE + + load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); + if (load_entropy_ok) { + RAND_seed(buf, sizeof(buf)); + } + + memwipe(buf, 0, sizeof(buf)); + + if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1) + return 0; + else + return -1; +} + +/** + * Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking + * for unit tests. + * + * This function is not allowed to fail; if it would fail to generate strong + * entropy, it must terminate the process instead. + **/ +MOCK_IMPL(void, +crypto_rand, (char *to, size_t n)) +{ + crypto_rand_unmocked(to, n); +} + +/** + * Write <b>n</b> bytes of strong random data to <b>to</b>. Most callers + * will want crypto_rand instead. + * + * This function is not allowed to fail; if it would fail to generate strong + * entropy, it must terminate the process instead. + **/ +void +crypto_rand_unmocked(char *to, size_t n) +{ + int r; + if (n == 0) + return; + + tor_assert(n < INT_MAX); + tor_assert(to); + r = RAND_bytes((unsigned char*)to, (int)n); + /* We consider a PRNG failure non-survivable. Let's assert so that we get a + * stack trace about where it happened. + */ + tor_assert(r >= 0); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + unsigned int val; + unsigned int cutoff; + tor_assert(max <= ((unsigned int)INT_MAX)+1); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT_MAX - (UINT_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * <b>min</b> MUST be in range [0, <b>max</b>). + * <b>max</b> MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + uint64_t val; + uint64_t cutoff; + tor_assert(max < UINT64_MAX); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT64_MAX - (UINT64_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * Generate and return a new random hostname starting with <b>prefix</b>, + * ending with <b>suffix</b>, and containing no fewer than + * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 + * characters. Does not check for failure. + * + * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE. + **/ +char * +crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, + const char *suffix) +{ + char *result, *rand_bytes; + int randlen, rand_bytes_len; + size_t resultlen, prefixlen; + + if (max_rand_len > MAX_DNS_LABEL_SIZE) + max_rand_len = MAX_DNS_LABEL_SIZE; + if (min_rand_len > max_rand_len) + min_rand_len = max_rand_len; + + randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1); + + prefixlen = strlen(prefix); + resultlen = prefixlen + strlen(suffix) + randlen + 16; + + rand_bytes_len = ((randlen*5)+7)/8; + if (rand_bytes_len % 5) + rand_bytes_len += 5 - (rand_bytes_len%5); + rand_bytes = tor_malloc(rand_bytes_len); + crypto_rand(rand_bytes, rand_bytes_len); + + result = tor_malloc(resultlen); + memcpy(result, prefix, prefixlen); + base32_encode(result+prefixlen, resultlen-prefixlen, + rand_bytes, rand_bytes_len); + tor_free(rand_bytes); + strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen)); + + return result; +} + +/** + * Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b> + * is empty. + **/ +void * +smartlist_choose(const smartlist_t *sl) +{ + int len = smartlist_len(sl); + if (len) + return smartlist_get(sl,crypto_rand_int(len)); + return NULL; /* no elements to choose from */ +} + +/** + * Scramble the elements of <b>sl</b> into a random order. + **/ +void +smartlist_shuffle(smartlist_t *sl) +{ + int i; + /* From the end of the list to the front, choose at random from the + positions we haven't looked at yet, and swap that position into the + current position. Remember to give "no swap" the same probability as + any other swap. */ + for (i = smartlist_len(sl)-1; i > 0; --i) { + int j = crypto_rand_int(i+1); + smartlist_swap(sl, i, j); + } +} + +/** Make sure that openssl is using its default PRNG. Return 1 if we had to + * adjust it; 0 otherwise. */ +int +crypto_force_rand_ssleay(void) +{ + RAND_METHOD *default_method; + default_method = RAND_OpenSSL(); + if (RAND_get_rand_method() != default_method) { + log_notice(LD_CRYPTO, "It appears that one of our engines has provided " + "a replacement the OpenSSL RNG. Resetting it to the default " + "implementation."); + RAND_set_rand_method(default_method); + return 1; + } + return 0; +} + +#endif /* !defined(CRYPTO_RAND_PRIVATE) */ + diff --git a/src/common/crypto_rand.h b/src/common/crypto_rand.h new file mode 100644 index 0000000000..bb02e51001 --- /dev/null +++ b/src/common/crypto_rand.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand.h + * + * \brief Common functions for using (pseudo-)random number generators. + **/ + +#ifndef TOR_CRYPTO_RAND_H +#define TOR_CRYPTO_RAND_H + +#include "torint.h" +#include "util.h" + +/* random numbers */ +int crypto_seed_rng(void) ATTR_WUR; +MOCK_DECL(void,crypto_rand,(char *to, size_t n)); +void crypto_rand_unmocked(char *to, size_t n); +void crypto_strongest_rand(uint8_t *out, size_t out_len); +int crypto_rand_int(unsigned int max); +int crypto_rand_int_range(unsigned int min, unsigned int max); +uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); +time_t crypto_rand_time_range(time_t min, time_t max); +uint64_t crypto_rand_uint64(uint64_t max); +double crypto_rand_double(void); +struct tor_weak_rng_t; +void crypto_seed_weak_rng(struct tor_weak_rng_t *rng); + +char *crypto_random_hostname(int min_rand_len, int max_rand_len, + const char *prefix, const char *suffix); + +struct smartlist_t; +void *smartlist_choose(const struct smartlist_t *sl); +void smartlist_shuffle(struct smartlist_t *sl); +int crypto_force_rand_ssleay(void); + +#ifdef CRYPTO_RAND_PRIVATE + +STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); + +#ifdef TOR_UNIT_TESTS +extern int break_strongest_rng_syscall; +extern int break_strongest_rng_fallback; +#endif +#endif /* defined(CRYPTO_RAND_PRIVATE) */ + +#endif /* !defined(TOR_CRYPTO_RAND_H) */ + diff --git a/src/common/crypto_rsa.c b/src/common/crypto_rsa.c new file mode 100644 index 0000000000..f66cdef3c5 --- /dev/null +++ b/src/common/crypto_rsa.c @@ -0,0 +1,1183 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rsa.c + * \brief Block of functions related with RSA utilities and operations. + **/ + +#include "crypto.h" +#include "crypto_curve25519.h" +#include "crypto_digest.h" +#include "crypto_format.h" +#include "compat_openssl.h" +#include "crypto_rand.h" +#include "crypto_rsa.h" +#include "crypto_util.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/err.h> +#include <openssl/rsa.h> +#include <openssl/pem.h> +#include <openssl/evp.h> +#include <openssl/engine.h> +#include <openssl/rand.h> +#include <openssl/bn.h> +#include <openssl/dh.h> +#include <openssl/conf.h> +#include <openssl/hmac.h> + +ENABLE_GCC_WARNING(redundant-decls) + +#include "torlog.h" +#include "util.h" +#include "util_format.h" + +/** Declaration for crypto_pk_t structure. */ +struct crypto_pk_t +{ + int refs; /**< reference count, so we don't have to copy keys */ + RSA *key; /**< The key itself */ +}; + +/** Log all pending crypto errors at level <b>severity</b>. Use + * <b>doing</b> to describe our current activities. + */ +static void +crypto_log_errors(int severity, const char *doing) +{ + unsigned long err; + const char *msg, *lib, *func; + while ((err = ERR_get_error()) != 0) { + msg = (const char*)ERR_reason_error_string(err); + lib = (const char*)ERR_lib_error_string(err); + func = (const char*)ERR_func_error_string(err); + if (!msg) msg = "(null)"; + if (!lib) lib = "(null)"; + if (!func) func = "(null)"; + if (BUG(!doing)) doing = "(null)"; + tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)", + doing, msg, lib, func); + } +} + +/** Return the number of bytes added by padding method <b>padding</b>. + */ +int +crypto_get_rsa_padding_overhead(int padding) +{ + switch (padding) + { + case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD; + default: tor_assert(0); return -1; // LCOV_EXCL_LINE + } +} + +/** Given a padding method <b>padding</b>, return the correct OpenSSL constant. + */ +int +crypto_get_rsa_padding(int padding) +{ + switch (padding) + { + case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING; + default: tor_assert(0); return -1; // LCOV_EXCL_LINE + } +} + +/** used internally: quicly validate a crypto_pk_t object as a private key. + * Return 1 iff the public key is valid, 0 if obviously invalid. + */ +static int +crypto_pk_private_ok(const crypto_pk_t *k) +{ +#ifdef OPENSSL_1_1_API + if (!k || !k->key) + return 0; + + const BIGNUM *p, *q; + RSA_get0_factors(k->key, &p, &q); + return p != NULL; /* XXX/yawning: Should we check q? */ +#else /* !(defined(OPENSSL_1_1_API)) */ + return k && k->key && k->key->p; +#endif /* defined(OPENSSL_1_1_API) */ +} + +/** used by tortls.c: wrap an RSA* in a crypto_pk_t. */ +crypto_pk_t * +crypto_new_pk_from_rsa_(RSA *rsa) +{ + crypto_pk_t *env; + tor_assert(rsa); + env = tor_malloc(sizeof(crypto_pk_t)); + env->refs = 1; + env->key = rsa; + return env; +} + +/** Helper, used by tor-gencert.c. Return the RSA from a + * crypto_pk_t. */ +RSA * +crypto_pk_get_rsa_(crypto_pk_t *env) +{ + return env->key; +} + +/** used by tortls.c: get an equivalent EVP_PKEY* for a crypto_pk_t. Iff + * private is set, include the private-key portion of the key. Return a valid + * pointer on success, and NULL on failure. */ +MOCK_IMPL(EVP_PKEY *, +crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) +{ + RSA *key = NULL; + EVP_PKEY *pkey = NULL; + tor_assert(env->key); + if (private) { + if (!(key = RSAPrivateKey_dup(env->key))) + goto error; + } else { + if (!(key = RSAPublicKey_dup(env->key))) + goto error; + } + if (!(pkey = EVP_PKEY_new())) + goto error; + if (!(EVP_PKEY_assign_RSA(pkey, key))) + goto error; + return pkey; + error: + if (pkey) + EVP_PKEY_free(pkey); + if (key) + RSA_free(key); + return NULL; +} + +/** Allocate and return storage for a public key. The key itself will not yet + * be set. + */ +MOCK_IMPL(crypto_pk_t *, +crypto_pk_new,(void)) +{ + RSA *rsa; + + rsa = RSA_new(); + tor_assert(rsa); + return crypto_new_pk_from_rsa_(rsa); +} + +/** Release a reference to an asymmetric key; when all the references + * are released, free the key. + */ +void +crypto_pk_free_(crypto_pk_t *env) +{ + if (!env) + return; + + if (--env->refs > 0) + return; + tor_assert(env->refs == 0); + + if (env->key) + RSA_free(env->key); + + tor_free(env); +} + +/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>. + * Return 0 on success, -1 on failure. + */ +MOCK_IMPL(int, +crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) +{ + tor_assert(env); + + if (env->key) { + RSA_free(env->key); + env->key = NULL; + } + + { + BIGNUM *e = BN_new(); + RSA *r = NULL; + if (!e) + goto done; + if (! BN_set_word(e, 65537)) + goto done; + r = RSA_new(); + if (!r) + goto done; + if (RSA_generate_key_ex(r, bits, e, NULL) == -1) + goto done; + + env->key = r; + r = NULL; + done: + if (e) + BN_clear_free(e); + if (r) + RSA_free(r); + } + + if (!env->key) { + crypto_log_errors(LOG_WARN, "generating RSA key"); + return -1; + } + + return 0; +} + +/** A PEM callback that always reports a failure to get a password */ +static int +pem_no_password_cb(char *buf, int size, int rwflag, void *u) +{ + (void)buf; + (void)size; + (void)rwflag; + (void)u; + return -1; +} + +/** Read a PEM-encoded private key from the <b>len</b>-byte string <b>s</b> + * into <b>env</b>. Return 0 on success, -1 on failure. If len is -1, + * the string is nul-terminated. + */ +int +crypto_pk_read_private_key_from_string(crypto_pk_t *env, + const char *s, ssize_t len) +{ + BIO *b; + + tor_assert(env); + tor_assert(s); + tor_assert(len < INT_MAX && len < SSIZE_T_CEILING); + + /* Create a read-only memory BIO, backed by the string 's' */ + b = BIO_new_mem_buf((char*)s, (int)len); + if (!b) + return -1; + + if (env->key) + RSA_free(env->key); + + env->key = PEM_read_bio_RSAPrivateKey(b,NULL,pem_no_password_cb,NULL); + + BIO_free(b); + + if (!env->key) { + crypto_log_errors(LOG_WARN, "Error parsing private key"); + return -1; + } + return 0; +} + +/** Read a PEM-encoded private key from the file named by + * <b>keyfile</b> into <b>env</b>. Return 0 on success, -1 on failure. + */ +int +crypto_pk_read_private_key_from_filename(crypto_pk_t *env, + const char *keyfile) +{ + char *contents; + int r; + + /* Read the file into a string. */ + contents = read_file_to_str(keyfile, 0, NULL); + if (!contents) { + log_warn(LD_CRYPTO, "Error reading private key from \"%s\"", keyfile); + return -1; + } + + /* Try to parse it. */ + r = crypto_pk_read_private_key_from_string(env, contents, -1); + memwipe(contents, 0, strlen(contents)); + tor_free(contents); + if (r) + return -1; /* read_private_key_from_string already warned, so we don't.*/ + + /* Make sure it's valid. */ + if (crypto_pk_check_key(env) <= 0) + return -1; + + return 0; +} + +/** Helper function to implement crypto_pk_write_*_key_to_string. Return 0 on + * success, -1 on failure. */ +static int +crypto_pk_write_key_to_string_impl(crypto_pk_t *env, char **dest, + size_t *len, int is_public) +{ + BUF_MEM *buf; + BIO *b; + int r; + + tor_assert(env); + tor_assert(env->key); + tor_assert(dest); + + b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ + if (!b) + return -1; + + /* Now you can treat b as if it were a file. Just use the + * PEM_*_bio_* functions instead of the non-bio variants. + */ + if (is_public) + r = PEM_write_bio_RSAPublicKey(b, env->key); + else + r = PEM_write_bio_RSAPrivateKey(b, env->key, NULL,NULL,0,NULL,NULL); + + if (!r) { + crypto_log_errors(LOG_WARN, "writing RSA key to string"); + BIO_free(b); + return -1; + } + + BIO_get_mem_ptr(b, &buf); + + *dest = tor_malloc(buf->length+1); + memcpy(*dest, buf->data, buf->length); + (*dest)[buf->length] = 0; /* nul terminate it */ + *len = buf->length; + + BIO_free(b); + + return 0; +} + +/** PEM-encode the public key portion of <b>env</b> and write it to a + * newly allocated string. On success, set *<b>dest</b> to the new + * string, *<b>len</b> to the string's length, and return 0. On + * failure, return -1. + */ +int +crypto_pk_write_public_key_to_string(crypto_pk_t *env, char **dest, + size_t *len) +{ + return crypto_pk_write_key_to_string_impl(env, dest, len, 1); +} + +/** PEM-encode the private key portion of <b>env</b> and write it to a + * newly allocated string. On success, set *<b>dest</b> to the new + * string, *<b>len</b> to the string's length, and return 0. On + * failure, return -1. + */ +int +crypto_pk_write_private_key_to_string(crypto_pk_t *env, char **dest, + size_t *len) +{ + return crypto_pk_write_key_to_string_impl(env, dest, len, 0); +} + +/** Read a PEM-encoded public key from the first <b>len</b> characters of + * <b>src</b>, and store the result in <b>env</b>. Return 0 on success, -1 on + * failure. + */ +int +crypto_pk_read_public_key_from_string(crypto_pk_t *env, const char *src, + size_t len) +{ + BIO *b; + + tor_assert(env); + tor_assert(src); + tor_assert(len<INT_MAX); + + b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ + if (!b) + return -1; + + BIO_write(b, src, (int)len); + + if (env->key) + RSA_free(env->key); + env->key = PEM_read_bio_RSAPublicKey(b, NULL, pem_no_password_cb, NULL); + BIO_free(b); + if (!env->key) { + crypto_log_errors(LOG_WARN, "reading public key from string"); + return -1; + } + + return 0; +} + +/** Write the private key from <b>env</b> into the file named by <b>fname</b>, + * PEM-encoded. Return 0 on success, -1 on failure. + */ +int +crypto_pk_write_private_key_to_filename(crypto_pk_t *env, + const char *fname) +{ + BIO *bio; + char *cp; + long len; + char *s; + int r; + + tor_assert(crypto_pk_private_ok(env)); + + if (!(bio = BIO_new(BIO_s_mem()))) + return -1; + if (PEM_write_bio_RSAPrivateKey(bio, env->key, NULL,NULL,0,NULL,NULL) + == 0) { + crypto_log_errors(LOG_WARN, "writing private key"); + BIO_free(bio); + return -1; + } + len = BIO_get_mem_data(bio, &cp); + tor_assert(len >= 0); + s = tor_malloc(len+1); + memcpy(s, cp, len); + s[len]='\0'; + r = write_str_to_file(fname, s, 0); + BIO_free(bio); + memwipe(s, 0, strlen(s)); + tor_free(s); + return r; +} + +/** Return true iff <b>env</b> has a valid key. + */ +int +crypto_pk_check_key(crypto_pk_t *env) +{ + int r; + tor_assert(env); + + r = RSA_check_key(env->key); + if (r <= 0) + crypto_log_errors(LOG_WARN,"checking RSA key"); + return r; +} + +/** Return true iff <b>key</b> contains the private-key portion of the RSA + * key. */ +int +crypto_pk_key_is_private(const crypto_pk_t *key) +{ + tor_assert(key); + return crypto_pk_private_ok(key); +} + +/** Return true iff <b>env</b> contains a public key whose public exponent + * equals 65537. + */ +int +crypto_pk_public_exponent_ok(crypto_pk_t *env) +{ + tor_assert(env); + tor_assert(env->key); + + const BIGNUM *e; + +#ifdef OPENSSL_1_1_API + const BIGNUM *n, *d; + RSA_get0_key(env->key, &n, &e, &d); +#else + e = env->key->e; +#endif /* defined(OPENSSL_1_1_API) */ + return BN_is_word(e, 65537); +} + +/** Compare the public-key components of a and b. Return less than 0 + * if a\<b, 0 if a==b, and greater than 0 if a\>b. A NULL key is + * considered to be less than all non-NULL keys, and equal to itself. + * + * Note that this may leak information about the keys through timing. + */ +int +crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b) +{ + int result; + char a_is_non_null = (a != NULL) && (a->key != NULL); + char b_is_non_null = (b != NULL) && (b->key != NULL); + char an_argument_is_null = !a_is_non_null | !b_is_non_null; + + result = tor_memcmp(&a_is_non_null, &b_is_non_null, sizeof(a_is_non_null)); + if (an_argument_is_null) + return result; + + const BIGNUM *a_n, *a_e; + const BIGNUM *b_n, *b_e; + +#ifdef OPENSSL_1_1_API + const BIGNUM *a_d, *b_d; + RSA_get0_key(a->key, &a_n, &a_e, &a_d); + RSA_get0_key(b->key, &b_n, &b_e, &b_d); +#else + a_n = a->key->n; + a_e = a->key->e; + b_n = b->key->n; + b_e = b->key->e; +#endif /* defined(OPENSSL_1_1_API) */ + + tor_assert(a_n != NULL && a_e != NULL); + tor_assert(b_n != NULL && b_e != NULL); + + result = BN_cmp(a_n, b_n); + if (result) + return result; + return BN_cmp(a_e, b_e); +} + +/** Compare the public-key components of a and b. Return non-zero iff + * a==b. A NULL key is considered to be distinct from all non-NULL + * keys, and equal to itself. + * + * Note that this may leak information about the keys through timing. + */ +int +crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b) +{ + return (crypto_pk_cmp_keys(a, b) == 0); +} + +/** Return the size of the public key modulus in <b>env</b>, in bytes. */ +size_t +crypto_pk_keysize(const crypto_pk_t *env) +{ + tor_assert(env); + tor_assert(env->key); + + return (size_t) RSA_size((RSA*)env->key); +} + +/** Return the size of the public key modulus of <b>env</b>, in bits. */ +int +crypto_pk_num_bits(crypto_pk_t *env) +{ + tor_assert(env); + tor_assert(env->key); + +#ifdef OPENSSL_1_1_API + /* It's so stupid that there's no other way to check that n is valid + * before calling RSA_bits(). + */ + const BIGNUM *n, *e, *d; + RSA_get0_key(env->key, &n, &e, &d); + tor_assert(n != NULL); + + return RSA_bits(env->key); +#else /* !(defined(OPENSSL_1_1_API)) */ + tor_assert(env->key->n); + return BN_num_bits(env->key->n); +#endif /* defined(OPENSSL_1_1_API) */ +} + +/** Increase the reference count of <b>env</b>, and return it. + */ +crypto_pk_t * +crypto_pk_dup_key(crypto_pk_t *env) +{ + tor_assert(env); + tor_assert(env->key); + + env->refs++; + return env; +} + +#ifdef TOR_UNIT_TESTS +/** For testing: replace dest with src. (Dest must have a refcount + * of 1) */ +void +crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src) +{ + tor_assert(dest); + tor_assert(dest->refs == 1); + tor_assert(src); + RSA_free(dest->key); + dest->key = RSAPrivateKey_dup(src->key); +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** Make a real honest-to-goodness copy of <b>env</b>, and return it. + * Returns NULL on failure. */ +crypto_pk_t * +crypto_pk_copy_full(crypto_pk_t *env) +{ + RSA *new_key; + int privatekey = 0; + tor_assert(env); + tor_assert(env->key); + + if (crypto_pk_private_ok(env)) { + new_key = RSAPrivateKey_dup(env->key); + privatekey = 1; + } else { + new_key = RSAPublicKey_dup(env->key); + } + if (!new_key) { + /* LCOV_EXCL_START + * + * We can't cause RSA*Key_dup() to fail, so we can't really test this. + */ + log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.", + privatekey?"private":"public"); + crypto_log_errors(LOG_ERR, + privatekey ? "Duplicating a private key" : + "Duplicating a public key"); + tor_fragile_assert(); + return NULL; + /* LCOV_EXCL_STOP */ + } + + return crypto_new_pk_from_rsa_(new_key); +} + +/** Perform a hybrid (public/secret) encryption on <b>fromlen</b> + * bytes of data from <b>from</b>, with padding type 'padding', + * storing the results on <b>to</b>. + * + * Returns the number of bytes written on success, -1 on failure. + * + * The encrypted data consists of: + * - The source data, padded and encrypted with the public key, if the + * padded source data is no longer than the public key, and <b>force</b> + * is false, OR + * - The beginning of the source data prefixed with a 16-byte symmetric key, + * padded and encrypted with the public key; followed by the rest of + * the source data encrypted in AES-CTR mode with the symmetric key. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. + */ +int +crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, + char *to, size_t tolen, + const char *from, + size_t fromlen, + int padding, int force) +{ + int overhead, outlen, r; + size_t pkeylen, symlen; + crypto_cipher_t *cipher = NULL; + char *buf = NULL; + + tor_assert(env); + tor_assert(from); + tor_assert(to); + tor_assert(fromlen < SIZE_T_CEILING); + + overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding)); + pkeylen = crypto_pk_keysize(env); + + if (!force && fromlen+overhead <= pkeylen) { + /* It all fits in a single encrypt. */ + return crypto_pk_public_encrypt(env,to, + tolen, + from,fromlen,padding); + } + tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN); + tor_assert(tolen >= pkeylen); + + char key[CIPHER_KEY_LEN]; + crypto_rand(key, sizeof(key)); /* generate a new key. */ + cipher = crypto_cipher_new(key); + + buf = tor_malloc(pkeylen+1); + memcpy(buf, key, CIPHER_KEY_LEN); + memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN); + + /* Length of symmetrically encrypted data. */ + symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN); + + outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding); + if (outlen!=(int)pkeylen) { + goto err; + } + r = crypto_cipher_encrypt(cipher, to+outlen, + from+pkeylen-overhead-CIPHER_KEY_LEN, symlen); + + if (r<0) goto err; + memwipe(buf, 0, pkeylen); + memwipe(key, 0, sizeof(key)); + tor_free(buf); + crypto_cipher_free(cipher); + tor_assert(outlen+symlen < INT_MAX); + return (int)(outlen + symlen); + err: + + memwipe(buf, 0, pkeylen); + memwipe(key, 0, sizeof(key)); + tor_free(buf); + crypto_cipher_free(cipher); + return -1; +} + +/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of + * bytes written on success, -1 on failure. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. + */ +int +crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, + char *to, + size_t tolen, + const char *from, + size_t fromlen, + int padding, int warnOnFailure) +{ + int outlen, r; + size_t pkeylen; + crypto_cipher_t *cipher = NULL; + char *buf = NULL; + + tor_assert(fromlen < SIZE_T_CEILING); + pkeylen = crypto_pk_keysize(env); + + if (fromlen <= pkeylen) { + return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding, + warnOnFailure); + } + + buf = tor_malloc(pkeylen); + outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding, + warnOnFailure); + if (outlen<0) { + log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO, + "Error decrypting public-key data"); + goto err; + } + if (outlen < CIPHER_KEY_LEN) { + log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO, + "No room for a symmetric key"); + goto err; + } + cipher = crypto_cipher_new(buf); + if (!cipher) { + goto err; + } + memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN); + outlen -= CIPHER_KEY_LEN; + tor_assert(tolen - outlen >= fromlen - pkeylen); + r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen); + if (r<0) + goto err; + memwipe(buf,0,pkeylen); + tor_free(buf); + crypto_cipher_free(cipher); + tor_assert(outlen + fromlen < INT_MAX); + return (int)(outlen + (fromlen-pkeylen)); + err: + memwipe(buf,0,pkeylen); + tor_free(buf); + crypto_cipher_free(cipher); + return -1; +} + +/** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key + * in <b>env</b>, using the padding method <b>padding</b>. On success, + * write the result to <b>to</b>, and return the number of bytes + * written. On failure, return -1. + * + * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be + * at least the length of the modulus of <b>env</b>. + */ +int +crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen, int padding) +{ + int r; + tor_assert(env); + tor_assert(from); + tor_assert(to); + tor_assert(fromlen<INT_MAX); + tor_assert(tolen >= crypto_pk_keysize(env)); + + r = RSA_public_encrypt((int)fromlen, + (unsigned char*)from, (unsigned char*)to, + env->key, crypto_get_rsa_padding(padding)); + if (r<0) { + crypto_log_errors(LOG_WARN, "performing RSA encryption"); + return -1; + } + return r; +} + +/** Decrypt <b>fromlen</b> bytes from <b>from</b> with the private key + * in <b>env</b>, using the padding method <b>padding</b>. On success, + * write the result to <b>to</b>, and return the number of bytes + * written. On failure, return -1. + * + * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be + * at least the length of the modulus of <b>env</b>. + */ +int +crypto_pk_private_decrypt(crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen, + int padding, int warnOnFailure) +{ + int r; + tor_assert(env); + tor_assert(from); + tor_assert(to); + tor_assert(env->key); + tor_assert(fromlen<INT_MAX); + tor_assert(tolen >= crypto_pk_keysize(env)); + if (!crypto_pk_key_is_private(env)) + /* Not a private key */ + return -1; + + r = RSA_private_decrypt((int)fromlen, + (unsigned char*)from, (unsigned char*)to, + env->key, crypto_get_rsa_padding(padding)); + + if (r<0) { + crypto_log_errors(warnOnFailure?LOG_WARN:LOG_DEBUG, + "performing RSA decryption"); + return -1; + } + return r; +} + +/** Check the signature in <b>from</b> (<b>fromlen</b> bytes long) with the + * public key in <b>env</b>, using PKCS1 padding. On success, write the + * signed data to <b>to</b>, and return the number of bytes written. + * On failure, return -1. + * + * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be + * at least the length of the modulus of <b>env</b>. + */ +MOCK_IMPL(int, +crypto_pk_public_checksig,(const crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen)) +{ + int r; + tor_assert(env); + tor_assert(from); + tor_assert(to); + tor_assert(fromlen < INT_MAX); + tor_assert(tolen >= crypto_pk_keysize(env)); + r = RSA_public_decrypt((int)fromlen, + (unsigned char*)from, (unsigned char*)to, + env->key, RSA_PKCS1_PADDING); + + if (r<0) { + crypto_log_errors(LOG_INFO, "checking RSA signature"); + return -1; + } + return r; +} + +/** Sign <b>fromlen</b> bytes of data from <b>from</b> with the private key in + * <b>env</b>, using PKCS1 padding. On success, write the signature to + * <b>to</b>, and return the number of bytes written. On failure, return + * -1. + * + * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be + * at least the length of the modulus of <b>env</b>. + */ +int +crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen) +{ + int r; + tor_assert(env); + tor_assert(from); + tor_assert(to); + tor_assert(fromlen < INT_MAX); + tor_assert(tolen >= crypto_pk_keysize(env)); + if (!crypto_pk_key_is_private(env)) + /* Not a private key */ + return -1; + + r = RSA_private_encrypt((int)fromlen, + (unsigned char*)from, (unsigned char*)to, + (RSA*)env->key, RSA_PKCS1_PADDING); + if (r<0) { + crypto_log_errors(LOG_WARN, "generating RSA signature"); + return -1; + } + return r; +} + +/** ASN.1-encode the public portion of <b>pk</b> into <b>dest</b>. + * Return -1 on error, or the number of characters used on success. + */ +int +crypto_pk_asn1_encode(const crypto_pk_t *pk, char *dest, size_t dest_len) +{ + int len; + unsigned char *buf = NULL; + + len = i2d_RSAPublicKey(pk->key, &buf); + if (len < 0 || buf == NULL) + return -1; + + if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) { + OPENSSL_free(buf); + return -1; + } + /* We don't encode directly into 'dest', because that would be illegal + * type-punning. (C99 is smarter than me, C99 is smarter than me...) + */ + memcpy(dest,buf,len); + OPENSSL_free(buf); + return len; +} + +/** Decode an ASN.1-encoded public key from <b>str</b>; return the result on + * success and NULL on failure. + */ +crypto_pk_t * +crypto_pk_asn1_decode(const char *str, size_t len) +{ + RSA *rsa; + unsigned char *buf; + const unsigned char *cp; + cp = buf = tor_malloc(len); + memcpy(buf,str,len); + rsa = d2i_RSAPublicKey(NULL, &cp, len); + tor_free(buf); + if (!rsa) { + crypto_log_errors(LOG_WARN,"decoding public key"); + return NULL; + } + return crypto_new_pk_from_rsa_(rsa); +} + +/** Given a private or public key <b>pk</b>, put a fingerprint of the + * public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 bytes of + * space). Return 0 on success, -1 on failure. + * + * Fingerprints are computed as the SHA1 digest of the ASN.1 encoding + * of the public key, converted to hexadecimal, in upper case, with a + * space after every four digits. + * + * If <b>add_space</b> is false, omit the spaces. + */ +int +crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out, int add_space) +{ + char digest[DIGEST_LEN]; + char hexdigest[HEX_DIGEST_LEN+1]; + if (crypto_pk_get_digest(pk, digest)) { + return -1; + } + base16_encode(hexdigest,sizeof(hexdigest),digest,DIGEST_LEN); + if (add_space) { + crypto_add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest); + } else { + strncpy(fp_out, hexdigest, HEX_DIGEST_LEN+1); + } + return 0; +} + +/** Given a private or public key <b>pk</b>, put a hashed fingerprint of + * the public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 + * bytes of space). Return 0 on success, -1 on failure. + * + * Hashed fingerprints are computed as the SHA1 digest of the SHA1 digest + * of the ASN.1 encoding of the public key, converted to hexadecimal, in + * upper case. + */ +int +crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out) +{ + char digest[DIGEST_LEN], hashed_digest[DIGEST_LEN]; + if (crypto_pk_get_digest(pk, digest)) { + return -1; + } + if (crypto_digest(hashed_digest, digest, DIGEST_LEN) < 0) { + return -1; + } + base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN); + return 0; +} + +/** Check a siglen-byte long signature at <b>sig</b> against + * <b>datalen</b> bytes of data at <b>data</b>, using the public key + * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for + * SHA1(data). Else return -1. + */ +MOCK_IMPL(int, +crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data, + size_t datalen, const char *sig, + size_t siglen)) +{ + char digest[DIGEST_LEN]; + char *buf; + size_t buflen; + int r; + + tor_assert(env); + tor_assert(data); + tor_assert(sig); + tor_assert(datalen < SIZE_T_CEILING); + tor_assert(siglen < SIZE_T_CEILING); + + if (crypto_digest(digest,data,datalen)<0) { + log_warn(LD_BUG, "couldn't compute digest"); + return -1; + } + buflen = crypto_pk_keysize(env); + buf = tor_malloc(buflen); + r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen); + if (r != DIGEST_LEN) { + log_warn(LD_CRYPTO, "Invalid signature"); + tor_free(buf); + return -1; + } + if (tor_memneq(buf, digest, DIGEST_LEN)) { + log_warn(LD_CRYPTO, "Signature mismatched with digest."); + tor_free(buf); + return -1; + } + tor_free(buf); + + return 0; +} + +/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at + * <b>from</b>; sign the data with the private key in <b>env</b>, and + * store it in <b>to</b>. Return the number of bytes written on + * success, and -1 on failure. + * + * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be + * at least the length of the modulus of <b>env</b>. + */ +int +crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen) +{ + int r; + char digest[DIGEST_LEN]; + if (crypto_digest(digest,from,fromlen)<0) + return -1; + r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN); + memwipe(digest, 0, sizeof(digest)); + return r; +} + +/** Given a private or public key <b>pk</b>, put a SHA1 hash of the + * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space). + * Return 0 on success, -1 on failure. + */ +int +crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out) +{ + char *buf; + size_t buflen; + int len; + int rv = -1; + + buflen = crypto_pk_keysize(pk)*2; + buf = tor_malloc(buflen); + len = crypto_pk_asn1_encode(pk, buf, buflen); + if (len < 0) + goto done; + + if (crypto_digest(digest_out, buf, len) < 0) + goto done; + + rv = 0; + done: + tor_free(buf); + return rv; +} + +/** Compute all digests of the DER encoding of <b>pk</b>, and store them + * in <b>digests_out</b>. Return 0 on success, -1 on failure. */ +int +crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out) +{ + char *buf; + size_t buflen; + int len; + int rv = -1; + + buflen = crypto_pk_keysize(pk)*2; + buf = tor_malloc(buflen); + len = crypto_pk_asn1_encode(pk, buf, buflen); + if (len < 0) + goto done; + + if (crypto_common_digests(digests_out, (char*)buf, len) < 0) + goto done; + + rv = 0; + done: + tor_free(buf); + return rv; +} + +/** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the + * Base64 encoding of the DER representation of the private key as a NUL + * terminated string, and return it via <b>priv_out</b>. Return 0 on + * success, -1 on failure. + * + * It is the caller's responsibility to sanitize and free the resulting buffer. + */ +int +crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out) +{ + unsigned char *der = NULL; + int der_len; + int ret = -1; + + *priv_out = NULL; + + der_len = i2d_RSAPrivateKey(pk->key, &der); + if (der_len < 0 || der == NULL) + return ret; + + size_t priv_len = base64_encode_size(der_len, 0) + 1; + char *priv = tor_malloc_zero(priv_len); + if (base64_encode(priv, priv_len, (char *)der, der_len, 0) >= 0) { + *priv_out = priv; + ret = 0; + } else { + tor_free(priv); + } + + memwipe(der, 0, der_len); + OPENSSL_free(der); + return ret; +} + +/** Given a string containing the Base64 encoded DER representation of the + * private key <b>str</b>, decode and return the result on success, or NULL + * on failure. + */ +crypto_pk_t * +crypto_pk_base64_decode(const char *str, size_t len) +{ + crypto_pk_t *pk = NULL; + + char *der = tor_malloc_zero(len + 1); + int der_len = base64_decode(der, len, str, len); + if (der_len <= 0) { + log_warn(LD_CRYPTO, "Stored RSA private key seems corrupted (base64)."); + goto out; + } + + const unsigned char *dp = (unsigned char*)der; /* Shut the compiler up. */ + RSA *rsa = d2i_RSAPrivateKey(NULL, &dp, der_len); + if (!rsa) { + crypto_log_errors(LOG_WARN, "decoding private key"); + goto out; + } + + pk = crypto_new_pk_from_rsa_(rsa); + + /* Make sure it's valid. */ + if (crypto_pk_check_key(pk) <= 0) { + crypto_pk_free(pk); + pk = NULL; + goto out; + } + + out: + memwipe(der, 0, len + 1); + tor_free(der); + return pk; +} + diff --git a/src/common/crypto_rsa.h b/src/common/crypto_rsa.h new file mode 100644 index 0000000000..e952089318 --- /dev/null +++ b/src/common/crypto_rsa.h @@ -0,0 +1,119 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rsa.h + * + * \brief Headers for crypto_rsa.c + **/ + +#ifndef TOR_CRYPTO_RSA_H +#define TOR_CRYPTO_RSA_H + +#include "orconfig.h" + +#include "crypto_digest.h" +#include <stdio.h> +#include "torint.h" +#include "testsupport.h" +#include "compat.h" +#include "util.h" +#include "torlog.h" + +/** Length of our public keys. */ +#define PK_BYTES (1024/8) + +/** Constant used to indicate OAEP padding for public-key encryption */ +#define PK_PKCS1_OAEP_PADDING 60002 + +/** Number of bytes added for PKCS1-OAEP padding. */ +#define PKCS1_OAEP_PADDING_OVERHEAD 42 + +/** A public key, or a public/private key-pair. */ +typedef struct crypto_pk_t crypto_pk_t; + +/* RSA environment setup */ +MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void)); +void crypto_pk_free_(crypto_pk_t *env); +#define crypto_pk_free(pk) FREE_AND_NULL(crypto_pk_t, crypto_pk_free_, (pk)) +int crypto_get_rsa_padding_overhead(int padding); +int crypto_get_rsa_padding(int padding); + +/* public key crypto */ +MOCK_DECL(int, crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)); +#define crypto_pk_generate_key(env) \ + crypto_pk_generate_key_with_bits((env), (PK_BYTES*8)) + +int crypto_pk_read_private_key_from_filename(crypto_pk_t *env, + const char *keyfile); +int crypto_pk_write_public_key_to_string(crypto_pk_t *env, + char **dest, size_t *len); +int crypto_pk_write_private_key_to_string(crypto_pk_t *env, + char **dest, size_t *len); +int crypto_pk_read_public_key_from_string(crypto_pk_t *env, + const char *src, size_t len); +int crypto_pk_read_private_key_from_string(crypto_pk_t *env, + const char *s, ssize_t len); +int crypto_pk_write_private_key_to_filename(crypto_pk_t *env, + const char *fname); + +int crypto_pk_check_key(crypto_pk_t *env); +int crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b); +int crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b); +size_t crypto_pk_keysize(const crypto_pk_t *env); +int crypto_pk_num_bits(crypto_pk_t *env); +crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig); +crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig); +int crypto_pk_key_is_private(const crypto_pk_t *key); +int crypto_pk_public_exponent_ok(crypto_pk_t *env); +int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen, + int padding, int force); +int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen, + int padding, int warnOnFailure); +int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen, int padding); +int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen, + int padding, int warnOnFailure); +MOCK_DECL(int, crypto_pk_public_checksig,(const crypto_pk_t *env, + char *to, size_t tolen, + const char *from, size_t fromlen)); +int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen); +int crypto_pk_asn1_encode(const crypto_pk_t *pk, char *dest, size_t dest_len); +crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len); +int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space); +int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out); + +MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env, + const char *data, size_t datalen, const char *sig, size_t siglen)); +int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen); +int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out); +int crypto_pk_get_common_digests(crypto_pk_t *pk, + common_digests_t *digests_out); +int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out); +crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len); + +/* Prototypes for private functions only used by tortls.c, crypto.c, and the + * unit tests. */ +struct rsa_st; +struct rsa_st *crypto_pk_get_rsa_(crypto_pk_t *env); +crypto_pk_t *crypto_new_pk_from_rsa_(struct rsa_st *rsa); +MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_evp_pkey_,(crypto_pk_t *env, + int private)); +struct evp_pkey_st; + +#ifdef TOR_UNIT_TESTS +void crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src); +#endif + +#endif + diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index 5dbd2ad91f..8543760ec5 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,10 +12,13 @@ #define CRYPTO_S2K_PRIVATE -#include "crypto.h" -#include "util.h" #include "compat.h" +#include "crypto.h" +#include "crypto_digest.h" +#include "crypto_rand.h" #include "crypto_s2k.h" +#include "crypto_util.h" +#include "util.h" #include <openssl/evp.h> @@ -86,9 +89,11 @@ secret_to_key_key_len(uint8_t type) return DIGEST_LEN; case S2K_TYPE_SCRYPT: return DIGEST256_LEN; + // LCOV_EXCL_START default: - tor_fragile_assert(); // LCOV_EXCL_LINE - return -1; // LCOV_EXCL_LINE + tor_fragile_assert(); + return -1; + // LCOV_EXCL_STOP } } @@ -169,9 +174,11 @@ make_specifier(uint8_t *spec_out, uint8_t type, unsigned flags) /* r = 8; p = 2. */ spec_out[SCRYPT_SPEC_LEN-1] = (3u << 4) | (1u << 0); break; + // LCOV_EXCL_START - we should have returned above. default: - tor_fragile_assert(); // LCOV_EXCL_LINE - we should have returned above. + tor_fragile_assert(); return S2K_BAD_ALGORITHM; + // LCOV_EXCL_STOP } return speclen; @@ -290,9 +297,9 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv != 0) return S2K_FAILED; return (int)key_out_len; -#else +#else /* !(defined(HAVE_SCRYPT)) */ return S2K_NO_SCRYPT_SUPPORT; -#endif +#endif /* defined(HAVE_SCRYPT) */ } default: return S2K_BAD_ALGORITHM; diff --git a/src/common/crypto_s2k.h b/src/common/crypto_s2k.h index 9b186450b1..849ff59ce8 100644 --- a/src/common/crypto_s2k.h +++ b/src/common/crypto_s2k.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_S2K_H_INCLUDED @@ -67,7 +67,7 @@ STATIC int secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, const uint8_t *spec, size_t spec_len, const char *secret, size_t secret_len, int type); -#endif +#endif /* defined(CRYPTO_S2K_PRIVATE) */ -#endif +#endif /* !defined(TOR_CRYPTO_S2K_H_INCLUDED) */ diff --git a/src/common/crypto_util.c b/src/common/crypto_util.c new file mode 100644 index 0000000000..b0d5b6b2f7 --- /dev/null +++ b/src/common/crypto_util.c @@ -0,0 +1,107 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_util.c + * + * \brief Common cryptographic utilities. + **/ + +#ifndef CRYPTO_UTIL_PRIVATE +#define CRYPTO_UTIL_PRIVATE + +#include "crypto_util.h" + +#include <string.h> + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#include <wincrypt.h> +#endif /* defined(_WIN32) */ + +#include "util.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/crypto.h> + +ENABLE_GCC_WARNING(redundant-decls) + +/** + * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to + * the value <b>byte</b>. + * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens. + * + * This function is preferable to memset, since many compilers will happily + * optimize out memset() when they can convince themselves that the data being + * cleared will never be read. + * + * Right now, our convention is to use this function when we are wiping data + * that's about to become inaccessible, such as stack buffers that are about + * to go out of scope or structures that are about to get freed. (In + * practice, it appears that the compilers we're currently using will optimize + * out the memset()s for stack-allocated buffers, but not those for + * about-to-be-freed structures. That could change, though, so we're being + * wary.) If there are live reads for the data, then you can just use + * memset(). + */ +void +memwipe(void *mem, uint8_t byte, size_t sz) +{ + if (sz == 0) { + return; + } + /* If sz is nonzero, then mem must not be NULL. */ + tor_assert(mem != NULL); + + /* Data this large is likely to be an underflow. */ + tor_assert(sz < SIZE_T_CEILING); + + /* Because whole-program-optimization exists, we may not be able to just + * have this function call "memset". A smart compiler could inline it, then + * eliminate dead memsets, and declare itself to be clever. */ + +#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) + /* Here's what you do on windows. */ + SecureZeroMemory(mem,sz); +#elif defined(HAVE_RTLSECUREZEROMEMORY) + RtlSecureZeroMemory(mem,sz); +#elif defined(HAVE_EXPLICIT_BZERO) + /* The BSDs provide this. */ + explicit_bzero(mem, sz); +#elif defined(HAVE_MEMSET_S) + /* This is in the C99 standard. */ + memset_s(mem, sz, 0, sz); +#else + /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk + * based on the pointer value, then uses that junk to update a global + * variable. It's an elaborate ruse to trick the compiler into not + * optimizing out the "wipe this memory" code. Read it if you like zany + * programming tricks! In later versions of Tor, we should look for better + * not-optimized-out memory wiping stuff... + * + * ...or maybe not. In practice, there are pure-asm implementations of + * OPENSSL_cleanse() on most platforms, which ought to do the job. + **/ + + OPENSSL_cleanse(mem, sz); +#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */ + + /* Just in case some caller of memwipe() is relying on getting a buffer + * filled with a particular value, fill the buffer. + * + * If this function gets inlined, this memset might get eliminated, but + * that's okay: We only care about this particular memset in the case where + * the caller should have been using memset(), and the memset() wouldn't get + * eliminated. In other words, this is here so that we won't break anything + * if somebody accidentally calls memwipe() instead of memset(). + **/ + memset(mem, byte, sz); +} + +#endif /* !defined(CRYPTO_UTIL_PRIVATE) */ + diff --git a/src/common/crypto_util.h b/src/common/crypto_util.h new file mode 100644 index 0000000000..922942b371 --- /dev/null +++ b/src/common/crypto_util.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_util.h + * + * \brief Common functions for cryptographic routines. + **/ + +#ifndef TOR_CRYPTO_UTIL_H +#define TOR_CRYPTO_UTIL_H + +#include "torint.h" + +/** OpenSSL-based utility functions. */ +void memwipe(void *mem, uint8_t byte, size_t sz); + +#ifdef CRYPTO_UTIL_PRIVATE +#ifdef TOR_UNIT_TESTS +#endif /* defined(TOR_UNIT_TESTS) */ +#endif /* defined(CRYPTO_UTIL_PRIVATE) */ + +#endif /* !defined(TOR_CRYPTO_UTIL_H) */ + diff --git a/src/common/di_ops.c b/src/common/di_ops.c index 4ed49e1164..90e9357c8e 100644 --- a/src/common/di_ops.c +++ b/src/common/di_ops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -86,7 +86,7 @@ tor_memcmp(const void *a, const void *b, size_t len) } return retval; -#endif /* timingsafe_memcmp */ +#endif /* defined(HAVE_TIMINGSAFE_MEMCMP) */ } /** @@ -148,7 +148,7 @@ struct di_digest256_map_t { /** Release all storage held in <b>map</b>, calling free_fn on each value * as we go. */ void -dimap_free(di_digest256_map_t *map, dimap_free_fn free_fn) +dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn) { while (map) { di_digest256_map_t *victim = map; @@ -238,7 +238,7 @@ gt_i64_timei(uint64_t a, uint64_t b) int res = diff >> 63; return res & 1; } -#endif +#endif /* SIZEOF_VOID_P == 8 */ /** * Given an array of list of <b>n_entries</b> uint64_t values, whose sum is diff --git a/src/common/di_ops.h b/src/common/di_ops.h index 0a154302bf..67d9c9f0df 100644 --- a/src/common/di_ops.h +++ b/src/common/di_ops.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -37,7 +37,12 @@ int safe_mem_is_zero(const void *mem, size_t sz); typedef struct di_digest256_map_t di_digest256_map_t; typedef void (*dimap_free_fn)(void *); -void dimap_free(di_digest256_map_t *map, dimap_free_fn free_fn); +void dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn); +#define dimap_free(map, free_fn) \ + do { \ + dimap_free_((map), (free_fn)); \ + (map) = NULL; \ + } while (0) void dimap_add_entry(di_digest256_map_t **map, const uint8_t *key, void *val); void *dimap_search(const di_digest256_map_t *map, const uint8_t *key, @@ -46,5 +51,5 @@ int select_array_member_cumulative_timei(const uint64_t *entries, int n_entries, uint64_t total, uint64_t rand_val); -#endif +#endif /* !defined(TOR_DI_OPS_H) */ diff --git a/src/common/handles.h b/src/common/handles.h index 1ee2322579..aef8cd89ef 100644 --- a/src/common/handles.h +++ b/src/common/handles.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -59,7 +59,7 @@ #define HANDLE_DECL(name, structname, linkage) \ typedef struct name ## _handle_t name ## _handle_t; \ linkage name ## _handle_t *name ## _handle_new(struct structname *object); \ - linkage void name ## _handle_free(name ## _handle_t *); \ + linkage void name ## _handle_free_(name ## _handle_t *); \ linkage struct structname *name ## _handle_get(name ## _handle_t *); \ linkage void name ## _handles_clear(struct structname *object); @@ -113,7 +113,7 @@ } \ \ linkage void \ - name ## _handle_free(struct name ## _handle_t *ref) \ + name ## _handle_free_(struct name ## _handle_t *ref) \ { \ if (! ref) return; \ name ## _handle_head_t *head = ref->head; \ @@ -149,5 +149,5 @@ } \ } -#endif /* TOR_HANDLE_H */ +#endif /* !defined(TOR_HANDLE_H) */ diff --git a/src/common/include.am b/src/common/include.am index cb307e9d5f..cfaf993674 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -74,7 +74,11 @@ LIBOR_CTIME_A_SRC = \ src/common/di_ops.c src_common_libor_ctime_a_SOURCES = $(LIBOR_CTIME_A_SRC) +if UNITTESTS_ENABLED src_common_libor_ctime_testing_a_SOURCES = $(LIBOR_CTIME_A_SRC) +else +src_common_libor_ctime_testing_a_SOURCES = +endif src_common_libor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) @@ -82,9 +86,11 @@ LIBOR_A_SRC = \ src/common/address.c \ src/common/address_set.c \ src/common/backtrace.c \ + src/common/buffers.c \ src/common/compat.c \ src/common/compat_threads.c \ src/common/compat_time.c \ + src/common/confline.c \ src/common/container.c \ src/common/log.c \ src/common/memarea.c \ @@ -94,6 +100,8 @@ LIBOR_A_SRC = \ src/common/util_format.c \ src/common/util_process.c \ src/common/sandbox.c \ + src/common/storagedir.c \ + src/common/token_bucket.c \ src/common/workqueue.c \ $(libor_extra_source) \ $(threads_impl_source) \ @@ -104,11 +112,21 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/buffers_tls.c \ + src/common/compress.c \ + src/common/compress_lzma.c \ + src/common/compress_none.c \ + src/common/compress_zlib.c \ + src/common/compress_zstd.c \ src/common/crypto.c \ + src/common/crypto_digest.c \ + src/common/crypto_format.c \ + src/common/crypto_openssl_mgt.c \ src/common/crypto_pwbox.c \ + src/common/crypto_rand.c \ + src/common/crypto_rsa.c \ src/common/crypto_s2k.c \ - src/common/crypto_format.c \ - src/common/torgzip.c \ + src/common/crypto_util.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -123,9 +141,15 @@ src_common_libor_a_SOURCES = $(LIBOR_A_SRC) src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SRC) +if UNITTESTS_ENABLED src_common_libor_testing_a_SOURCES = $(LIBOR_A_SRC) src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SRC) +else +src_common_libor_testing_a_SOURCES = +src_common_libor_crypto_testing_a_SOURCES = +src_common_libor_event_testing_a_SOURCES = +endif src_common_libor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_common_libor_crypto_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) @@ -138,6 +162,8 @@ COMMONHEADERS = \ src/common/address.h \ src/common/address_set.h \ src/common/backtrace.h \ + src/common/buffers.h \ + src/common/buffers_tls.h \ src/common/aes.h \ src/common/ciphers.inc \ src/common/compat.h \ @@ -145,13 +171,24 @@ COMMONHEADERS = \ src/common/compat_openssl.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress.h \ + src/common/compress_lzma.h \ + src/common/compress_none.h \ + src/common/compress_zlib.h \ + src/common/compress_zstd.h \ + src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ + src/common/crypto_digest.h \ src/common/crypto_curve25519.h \ src/common/crypto_ed25519.h \ src/common/crypto_format.h \ + src/common/crypto_openssl_mgt.h \ src/common/crypto_pwbox.h \ + src/common/crypto_rand.h \ + src/common/crypto_rsa.h \ src/common/crypto_s2k.h \ + src/common/crypto_util.h \ src/common/di_ops.h \ src/common/handles.h \ src/common/memarea.h \ @@ -159,9 +196,10 @@ COMMONHEADERS = \ src/common/procmon.h \ src/common/pubsub.h \ src/common/sandbox.h \ + src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ - src/common/torgzip.h \ + src/common/token_bucket.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/log.c b/src/common/log.c index 4db1c9f0d0..ebd50f62d3 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -35,6 +35,9 @@ #define LOG_PRIVATE #include "torlog.h" #include "container.h" +#ifdef HAVE_ANDROID_LOG_H +#include <android/log.h> +#endif // HAVE_ANDROID_LOG_H. /** Given a severity, yields an index into log_severity_list_t.masks to use * for that severity. */ @@ -49,6 +52,13 @@ #define raw_assert(x) assert(x) // assert OK +/** Defining compile-time constants for Tor log levels (used by the Rust + * log wrapper at src/rust/tor_log) */ +const int LOG_WARN_ = LOG_WARN; +const int LOG_NOTICE_ = LOG_NOTICE; +const log_domain_mask_t LD_GENERAL_ = LD_GENERAL; +const log_domain_mask_t LD_NET_ = LD_NET; + /** Information for a single logfile; only used in log.c */ typedef struct logfile_t { struct logfile_t *next; /**< Next logfile_t in the linked list. */ @@ -58,12 +68,16 @@ typedef struct logfile_t { int needs_close; /**< Boolean: true if the stream gets closed on shutdown. */ int is_temporary; /**< Boolean: close after initializing logging subsystem.*/ int is_syslog; /**< Boolean: send messages to syslog. */ + int is_android; /**< Boolean: send messages to Android's log subsystem. */ + char *android_tag; /**< Identity Tag used in Android's log subsystem. */ log_callback callback; /**< If not NULL, send messages to this function. */ log_severity_list_t *severities; /**< Which severity of messages should we * log for each log domain? */ } logfile_t; -static void log_free(logfile_t *victim); +static void log_free_(logfile_t *victim); +#define log_free(lg) \ + FREE_AND_NULL(logfile_t, log_free_, (lg)) /** Helper: map a log severity to descriptive string. */ static inline const char * @@ -101,6 +115,33 @@ should_log_function_name(log_domain_mask_t domain, int severity) } } +#ifdef HAVE_ANDROID_LOG_H +/** Helper function to convert Tor's log severity into the matching + * Android log priority. + */ +static int +severity_to_android_log_priority(int severity) +{ + switch (severity) { + case LOG_DEBUG: + return ANDROID_LOG_VERBOSE; + case LOG_INFO: + return ANDROID_LOG_DEBUG; + case LOG_NOTICE: + return ANDROID_LOG_INFO; + case LOG_WARN: + return ANDROID_LOG_WARN; + case LOG_ERR: + return ANDROID_LOG_ERROR; + default: + // LCOV_EXCL_START + raw_assert(0); + return 0; + // LCOV_EXCL_STOP + } +} +#endif // HAVE_ANDROID_LOG_H. + /** A mutex to guard changes to logfiles and logging. */ static tor_mutex_t log_mutex; /** True iff we have initialized log_mutex */ @@ -129,6 +170,9 @@ typedef struct pending_log_message_t { /** Log messages waiting to be replayed onto callback-based logs */ static smartlist_t *pending_cb_messages = NULL; +/** Callback to invoke when pending_cb_messages becomes nonempty. */ +static pending_callback_callback pending_cb_cb = NULL; + /** Log messages waiting to be replayed once the logging system is initialized. */ static smartlist_t *pending_startup_messages = NULL; @@ -191,6 +235,30 @@ log_set_application_name(const char *name) appname = name ? tor_strdup(name) : NULL; } +/** Return true if some of the running logs might be interested in a log + * message of the given severity in the given domains. If this function + * returns true, the log message might be ignored anyway, but if it returns + * false, it is definitely_ safe not to log the message. */ +int +log_message_is_interesting(int severity, log_domain_mask_t domain) +{ + (void) domain; + return (severity <= log_global_min_severity_); +} + +/** + * As tor_log, but takes an optional function name, and does not treat its + * <b>string</b> as a printf format. + * + * For use by Rust integration. + */ +void +tor_log_string(int severity, log_domain_mask_t domain, + const char *function, const char *string) +{ + log_fn_(severity, domain, function, "%s", string); +} + /** Log time granularity in milliseconds. */ static int log_time_granularity = 1; @@ -385,9 +453,12 @@ pending_log_message_new(int severity, log_domain_mask_t domain, return m; } +#define pending_log_message_free(msg) \ + FREE_AND_NULL(pending_log_message_t, pending_log_message_free_, (msg)) + /** Release all storage held by <b>msg</b>. */ static void -pending_log_message_free(pending_log_message_t *msg) +pending_log_message_free_(pending_log_message_t *msg) { if (!msg) return; @@ -396,6 +467,16 @@ pending_log_message_free(pending_log_message_t *msg) tor_free(msg); } +/** Helper function: returns true iff the log file, given in <b>lf</b>, is + * handled externally via the system log API, the Android logging API, or is an + * external callback function. */ +static inline int +logfile_is_external(const logfile_t *lf) +{ + raw_assert(lf); + return lf->is_syslog || lf->is_android || lf->callback; +} + /** Return true iff <b>lf</b> would like to receive a message with the * specified <b>severity</b> in the specified <b>domain</b>. */ @@ -406,7 +487,7 @@ logfile_wants_message(const logfile_t *lf, int severity, if (! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) { return 0; } - if (! (lf->fd >= 0 || lf->is_syslog || lf->callback)) { + if (! (lf->fd >= 0 || logfile_is_external(lf))) { return 0; } if (lf->seems_dead) { @@ -444,17 +525,25 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, if (m != msg_after_prefix) { tor_free(m); } -#else +#else /* !(defined(MAXLINE)) */ /* We have syslog but not MAXLINE. That's promising! */ syslog(severity, "%s", msg_after_prefix); -#endif -#endif +#endif /* defined(MAXLINE) */ +#endif /* defined(HAVE_SYSLOG_H) */ + } else if (lf->is_android) { +#ifdef HAVE_ANDROID_LOG_H + int priority = severity_to_android_log_priority(severity); + __android_log_write(priority, lf->android_tag, msg_after_prefix); +#endif // HAVE_ANDROID_LOG_H. } else if (lf->callback) { if (domain & LD_NOCB) { if (!*callbacks_deferred && pending_cb_messages) { smartlist_add(pending_cb_messages, pending_log_message_new(severity,domain,NULL,msg_after_prefix)); *callbacks_deferred = 1; + if (smartlist_len(pending_cb_messages) == 1 && pending_cb_cb) { + pending_cb_cb(); + } } } else { lf->callback(severity, domain, msg_after_prefix); @@ -641,8 +730,8 @@ tor_log_update_sigsafe_err_fds(void) /* Don't try callback to the control port, or syslogs: We can't * do them from a signal handler. Don't try stdout: we always do stderr. */ - if (lf->is_temporary || lf->is_syslog || - lf->callback || lf->seems_dead || lf->fd < 0) + if (lf->is_temporary || logfile_is_external(lf) + || lf->seems_dead || lf->fd < 0) continue; if (lf->severities->masks[SEVERITY_MASK_IDX(LOG_ERR)] & (LD_BUG|LD_GENERAL)) { @@ -678,11 +767,11 @@ tor_log_get_logfile_names(smartlist_t *out) LOCK_LOGS(); for (lf = logfiles; lf; lf = lf->next) { - if (lf->is_temporary || lf->is_syslog || lf->callback) + if (lf->is_temporary || logfile_is_external(lf)) continue; if (lf->filename == NULL) continue; - smartlist_add(out, tor_strdup(lf->filename)); + smartlist_add_strdup(out, lf->filename); } UNLOCK_LOGS(); @@ -721,12 +810,13 @@ log_fn_ratelim_(ratelim_t *ratelim, int severity, log_domain_mask_t domain, /** Free all storage held by <b>victim</b>. */ static void -log_free(logfile_t *victim) +log_free_(logfile_t *victim) { if (!victim) return; tor_free(victim->severities); tor_free(victim->filename); + tor_free(victim->android_tag); tor_free(victim); } @@ -741,6 +831,7 @@ logs_free_all(void) logfiles = NULL; messages = pending_cb_messages; pending_cb_messages = NULL; + pending_cb_cb = NULL; messages2 = pending_startup_messages; pending_startup_messages = NULL; UNLOCK_LOGS(); @@ -807,7 +898,7 @@ close_log(logfile_t *victim) /* There are no other syslogs; close the logging facility. */ closelog(); } -#endif +#endif /* defined(HAVE_SYSLOG_H) */ } } @@ -904,6 +995,24 @@ add_temp_log(int min_severity) } /** + * Register "cb" as the callback to call when there are new pending log + * callbacks to be flushed with flush_pending_log_callbacks(). + * + * Note that this callback, if present, can be invoked from any thread. + * + * This callback must not log. + * + * It is intentional that this function contains the name "callback" twice: it + * sets a "callback" to be called on the condition that there is a "pending + * callback". + **/ +void +logs_set_pending_callback_callback(pending_callback_callback cb) +{ + pending_cb_cb = cb; +} + +/** * Add a log handler to send messages in <b>severity</b> * to the function <b>cb</b>. */ @@ -1086,7 +1195,7 @@ add_file_log(const log_severity_list_t *severity, const char *filename, int open_flags = O_WRONLY|O_CREAT; open_flags |= truncate_log ? O_TRUNC : O_APPEND; - fd = tor_open_cloexec(filename, open_flags, 0644); + fd = tor_open_cloexec(filename, open_flags, 0640); if (fd<0) return -1; if (tor_fd_seekend(fd)<0) { @@ -1144,7 +1253,40 @@ add_syslog_log(const log_severity_list_t *severity, UNLOCK_LOGS(); return 0; } -#endif +#endif /* defined(HAVE_SYSLOG_H) */ + +#ifdef HAVE_ANDROID_LOG_H +/** + * Add a log handler to send messages to the Android platform log facility. + */ +int +add_android_log(const log_severity_list_t *severity, + const char *android_tag) +{ + logfile_t *lf = NULL; + + lf = tor_malloc_zero(sizeof(logfile_t)); + lf->fd = -1; + lf->severities = tor_memdup(severity, sizeof(log_severity_list_t)); + lf->filename = tor_strdup("<android>"); + lf->is_android = 1; + + if (android_tag == NULL) + lf->android_tag = tor_strdup("Tor"); + else { + char buf[256]; + tor_snprintf(buf, sizeof(buf), "Tor-%s", android_tag); + lf->android_tag = tor_strdup(buf); + } + + LOCK_LOGS(); + lf->next = logfiles; + logfiles = lf; + log_global_min_severity_ = get_min_log_level(); + UNLOCK_LOGS(); + return 0; +} +#endif // HAVE_ANDROID_LOG_H. /** If <b>level</b> is a valid log severity, return the corresponding * numeric value. Otherwise, return -1. */ @@ -1172,12 +1314,15 @@ log_level_to_string(int level) } /** NULL-terminated array of names for log domains such that domain_list[dom] - * is a description of <b>dom</b>. */ + * is a description of <b>dom</b>. + * + * Remember to update doc/tor.1.txt if you modify this list. + * */ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, @@ -1313,16 +1458,15 @@ parse_log_severity_config(const char **cfg_ptr, if (!strcasecmpstart(cfg, "file") || !strcasecmpstart(cfg, "stderr") || !strcasecmpstart(cfg, "stdout") || - !strcasecmpstart(cfg, "syslog")) { + !strcasecmpstart(cfg, "syslog") || + !strcasecmpstart(cfg, "android")) { goto done; } if (got_an_unqualified_range > 1) return -1; - space = strchr(cfg, ' '); + space = find_whitespace(cfg); dash = strchr(cfg, '-'); - if (!space) - space = strchr(cfg, '\0'); if (dash && dash < space) { sev_lo = tor_strndup(cfg, dash-cfg); sev_hi = tor_strndup(dash+1, space-(dash+1)); diff --git a/src/common/memarea.c b/src/common/memarea.c index 7d16b702e3..68c1625fe4 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** \file memarea.c @@ -7,11 +7,15 @@ */ #include "orconfig.h" +#include <stddef.h> #include <stdlib.h> #include "memarea.h" #include "util.h" #include "compat.h" #include "torlog.h" +#include "container.h" + +#ifndef DISABLE_MEMORY_SENTINELS /** If true, we try to detect any attempts to write beyond the length of a * memarea. */ @@ -29,7 +33,7 @@ #define MEMAREA_ALIGN_MASK ((uintptr_t)7) #else #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." -#endif +#endif /* MEMAREA_ALIGN == 4 || ... */ #if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) #define USE_ALIGNED_ATTRIBUTE @@ -37,7 +41,7 @@ #define U_MEM mem #else #define U_MEM u.mem -#endif +#endif /* defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) */ #ifdef USE_SENTINELS /** Magic value that we stick at the end of a memarea so we can make sure @@ -57,11 +61,11 @@ uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END -#else +#else /* !(defined(USE_SENTINELS)) */ #define SENTINEL_LEN 0 #define SET_SENTINEL(chunk) STMT_NIL #define CHECK_SENTINEL(chunk) STMT_NIL -#endif +#endif /* defined(USE_SENTINELS) */ /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */ static inline void * @@ -93,12 +97,12 @@ typedef struct memarea_chunk_t { void *void_for_alignment_; /**< Dummy; used to make sure mem is aligned. */ } u; /**< Union used to enforce alignment when we don't have support for * doing it right. */ -#endif +#endif /* defined(USE_ALIGNED_ATTRIBUTE) */ } memarea_chunk_t; /** How many bytes are needed for overhead before we get to the memory part * of a chunk? */ -#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, U_MEM) +#define CHUNK_HEADER_SIZE offsetof(memarea_chunk_t, U_MEM) /** What's the smallest that we'll allocate a chunk? */ #define CHUNK_SIZE 4096 @@ -149,7 +153,7 @@ memarea_new(void) /** Free <b>area</b>, invalidating all pointers returned from memarea_alloc() * and friends for this area */ void -memarea_drop_all(memarea_t *area) +memarea_drop_all_(memarea_t *area) { memarea_chunk_t *chunk, *next; for (chunk = area->first; chunk; chunk = next) { @@ -304,3 +308,91 @@ memarea_assert_ok(memarea_t *area) } } +#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */ + +struct memarea_t { + smartlist_t *pieces; +}; + +memarea_t * +memarea_new(void) +{ + memarea_t *ma = tor_malloc_zero(sizeof(memarea_t)); + ma->pieces = smartlist_new(); + return ma; +} +void +memarea_drop_all_(memarea_t *area) +{ + memarea_clear(area); + smartlist_free(area->pieces); + tor_free(area); +} +void +memarea_clear(memarea_t *area) +{ + SMARTLIST_FOREACH(area->pieces, void *, p, tor_free_(p)); + smartlist_clear(area->pieces); +} +int +memarea_owns_ptr(const memarea_t *area, const void *ptr) +{ + SMARTLIST_FOREACH(area->pieces, const void *, p, if (ptr == p) return 1;); + return 0; +} + +void * +memarea_alloc(memarea_t *area, size_t sz) +{ + void *result = tor_malloc(sz); + smartlist_add(area->pieces, result); + return result; +} + +void * +memarea_alloc_zero(memarea_t *area, size_t sz) +{ + void *result = tor_malloc_zero(sz); + smartlist_add(area->pieces, result); + return result; +} +void * +memarea_memdup(memarea_t *area, const void *s, size_t n) +{ + void *r = memarea_alloc(area, n); + memcpy(r, s, n); + return r; +} +char * +memarea_strdup(memarea_t *area, const char *s) +{ + size_t n = strlen(s); + char *r = memarea_alloc(area, n+1); + memcpy(r, s, n); + r[n] = 0; + return r; +} +char * +memarea_strndup(memarea_t *area, const char *s, size_t n) +{ + size_t ln = strnlen(s, n); + char *r = memarea_alloc(area, ln+1); + memcpy(r, s, ln); + r[ln] = 0; + return r; +} +void +memarea_get_stats(memarea_t *area, + size_t *allocated_out, size_t *used_out) +{ + (void)area; + *allocated_out = *used_out = 128; +} +void +memarea_assert_ok(memarea_t *area) +{ + (void)area; +} + +#endif /* !defined(DISABLE_MEMORY_SENTINELS) */ + diff --git a/src/common/memarea.h b/src/common/memarea.h index 85bca51ad3..5207e8a5bd 100644 --- a/src/common/memarea.h +++ b/src/common/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Tor dependencies */ @@ -8,7 +8,12 @@ typedef struct memarea_t memarea_t; memarea_t *memarea_new(void); -void memarea_drop_all(memarea_t *area); +void memarea_drop_all_(memarea_t *area); +#define memarea_drop_all(area) \ + do { \ + memarea_drop_all_(area); \ + (area) = NULL; \ + } while (0) void memarea_clear(memarea_t *area); int memarea_owns_ptr(const memarea_t *area, const void *ptr); void *memarea_alloc(memarea_t *area, size_t sz); @@ -20,5 +25,5 @@ void memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out); void memarea_assert_ok(memarea_t *area); -#endif +#endif /* !defined(TOR_MEMAREA_H) */ diff --git a/src/common/procmon.c b/src/common/procmon.c index c485c760c7..73c14cd584 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,8 +10,6 @@ #include "util.h" -#include <event2/event.h> - #ifdef HAVE_SIGNAL_H #include <signal.h> #endif @@ -36,7 +34,7 @@ typedef int pid_t; #define PID_T_FORMAT I64_FORMAT #else #error Unknown: SIZEOF_PID_T -#endif +#endif /* (0 == SIZEOF_PID_T) && defined(_WIN32) || ... */ /* Define to 1 if process-termination monitors on this OS and Libevent version must poll for process termination themselves. */ @@ -44,7 +42,7 @@ typedef int pid_t; /* Currently we need to poll in some way on all systems. */ #ifdef PROCMON_POLLS -static void tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, +static void tor_process_monitor_poll_cb(periodic_timer_t *ev, void *procmon_); #endif @@ -71,7 +69,7 @@ parse_process_specifier(const char *process_spec, /* If we're lucky, long will turn out to be large enough to hold a * PID everywhere that Tor runs. */ - pid_l = tor_parse_long(process_spec, 0, 1, LONG_MAX, &pid_ok, &pspec_next); + pid_l = tor_parse_long(process_spec, 10, 1, LONG_MAX, &pid_ok, &pspec_next); /* Reserve room in the ‘process specifier’ for additional * (platform-specific) identifying information beyond the PID, to @@ -114,7 +112,7 @@ struct tor_process_monitor_t { HANDLE hproc; /* XXXX We should have Libevent watch hproc for us, * if/when some version of Libevent can be told to do so. */ -#endif +#endif /* defined(_WIN32) */ /* XXXX On Linux, we can and should receive the 22nd * (space-delimited) field (‘starttime’) of /proc/$PID/stat from the @@ -136,7 +134,7 @@ struct tor_process_monitor_t { /** A Libevent event structure, to either poll for the process's * existence or receive a notification when the process ends. */ - struct event *e; + periodic_timer_t *e; /** A callback to be called when the process ends. */ tor_procmon_callback_t cb; @@ -159,9 +157,6 @@ tor_validate_process_specifier(const char *process_spec, return parse_process_specifier(process_spec, &ppspec, msg); } -/* XXXX we should use periodic_timer_new() for this stuff */ -#define PERIODIC_TIMER_FLAGS EV_PERSIST - /* DOCDOC poll_interval_tv */ static const struct timeval poll_interval_tv = {15, 0}; @@ -219,22 +214,18 @@ tor_process_monitor_new(struct event_base *base, "try again later.", procmon->pid); } -#endif +#endif /* defined(_WIN32) */ procmon->cb = cb; procmon->cb_arg = cb_arg; #ifdef PROCMON_POLLS - procmon->e = tor_event_new(base, -1 /* no FD */, PERIODIC_TIMER_FLAGS, - tor_process_monitor_poll_cb, procmon); - /* Note: If you port this file to plain Libevent 2, check that - * procmon->e is non-NULL. We don't need to here because - * tor_evtimer_new never returns NULL. */ - - evtimer_add(procmon->e, &poll_interval_tv); -#else + procmon->e = periodic_timer_new(base, + &poll_interval_tv, + tor_process_monitor_poll_cb, procmon); +#else /* !(defined(PROCMON_POLLS)) */ #error OOPS? -#endif +#endif /* defined(PROCMON_POLLS) */ return procmon; err: @@ -246,14 +237,12 @@ tor_process_monitor_new(struct event_base *base, /** Libevent callback to poll for the existence of the process * monitored by <b>procmon_</b>. */ static void -tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, - void *procmon_) +tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_) { + (void)event; tor_process_monitor_t *procmon = (tor_process_monitor_t *)(procmon_); int its_dead_jim; - (void)unused1; (void)unused2; - tor_assert(procmon != NULL); #ifdef _WIN32 @@ -306,11 +295,11 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, tor_free(errmsg); } } -#else +#else /* !(defined(_WIN32)) */ /* Unix makes this part easy, if a bit racy. */ its_dead_jim = kill(procmon->pid, 0); its_dead_jim = its_dead_jim && (errno == ESRCH); -#endif +#endif /* defined(_WIN32) */ tor_log(its_dead_jim ? LOG_NOTICE : LOG_INFO, procmon->log_domain, "Monitored process "PID_T_FORMAT" is %s.", @@ -321,11 +310,11 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, procmon->cb(procmon->cb_arg); } } -#endif +#endif /* defined(PROCMON_POLLS) */ /** Free the process-termination monitor <b>procmon</b>. */ void -tor_process_monitor_free(tor_process_monitor_t *procmon) +tor_process_monitor_free_(tor_process_monitor_t *procmon) { if (procmon == NULL) return; @@ -336,7 +325,7 @@ tor_process_monitor_free(tor_process_monitor_t *procmon) #endif if (procmon->e != NULL) - tor_event_free(procmon->e); + periodic_timer_free(procmon->e); tor_free(procmon); } diff --git a/src/common/procmon.h b/src/common/procmon.h index 49ead24092..63777e4111 100644 --- a/src/common/procmon.h +++ b/src/common/procmon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,7 +27,9 @@ tor_process_monitor_t *tor_process_monitor_new(struct event_base *base, tor_procmon_callback_t cb, void *cb_arg, const char **msg); -void tor_process_monitor_free(tor_process_monitor_t *procmon); +void tor_process_monitor_free_(tor_process_monitor_t *procmon); +#define tor_process_monitor_free(procmon) \ + FREE_AND_NULL(tor_process_monitor_t, tor_process_monitor_free_, (procmon)) -#endif +#endif /* !defined(TOR_PROCMON_H) */ diff --git a/src/common/pubsub.c b/src/common/pubsub.c index b3faf40e00..336e8a6e7f 100644 --- a/src/common/pubsub.c +++ b/src/common/pubsub.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.h b/src/common/pubsub.h index bbb4f02a42..2bee3af085 100644 --- a/src/common/pubsub.h +++ b/src/common/pubsub.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -175,5 +175,5 @@ int pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn, pubsub_clear_(&name##_topic_); \ } -#endif /* TOR_PUBSUB_H */ +#endif /* !defined(TOR_PUBSUB_H) */ diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 0a972d496b..ca7803ac9c 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,10 +17,16 @@ * with the libevent fix. */ #define _LARGEFILE64_SOURCE -#endif +#endif /* !defined(_LARGEFILE64_SOURCE) */ -/** Malloc mprotect limit in bytes. */ -#define MALLOC_MP_LIM 1048576 +/** Malloc mprotect limit in bytes. + * + * 28/06/2017: This value was increased from 16 MB to 20 MB after we introduced + * LZMA support in Tor (0.3.1.1-alpha). We limit our LZMA coder to 16 MB, but + * liblzma have a small overhead that we need to compensate for to avoid being + * killed by the sandbox. + */ +#define MALLOC_MP_LIM (20*1024*1024) #include <stdio.h> #include <string.h> @@ -74,7 +80,7 @@ #define USE_BACKTRACE #define EXPOSE_CLEAN_BACKTRACE #include "backtrace.h" -#endif +#endif /* defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && ... */ #ifdef USE_BACKTRACE #include <execinfo.h> @@ -100,7 +106,12 @@ #define M_SYSCALL arm_r7 -#endif +#elif defined(__aarch64__) && defined(__LP64__) + +#define REG_SYSCALL 8 +#define M_SYSCALL regs[REG_SYSCALL] + +#endif /* defined(__i386__) || ... */ /**Determines if at least one sandbox is active.*/ static int sandbox_active = 0; @@ -151,6 +162,7 @@ static int filter_nopar_gen[] = { SCMP_SYS(fstat64), #endif SCMP_SYS(futex), + SCMP_SYS(getdents), SCMP_SYS(getdents64), SCMP_SYS(getegid), #ifdef __NR_getegid32 @@ -293,37 +305,6 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return rc; } -#if 0 -/** - * Function responsible for setting up the execve syscall for - * the seccomp filter sandbox. - */ -static int -sb_execve(scmp_filter_ctx ctx, sandbox_cfg_t *filter) -{ - int rc; - sandbox_cfg_t *elem = NULL; - - // for each dynamic parameter filters - for (elem = filter; elem != NULL; elem = elem->next) { - smp_param_t *param = elem->param; - - if (param != NULL && param->prot == 1 && param->syscall - == SCMP_SYS(execve)) { - rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(execve), - SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); - if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add execve syscall, received " - "libseccomp error %d", rc); - return rc; - } - } - } - - return 0; -} -#endif - /** * Function responsible for setting up the time syscall for * the seccomp filter sandbox. @@ -337,7 +318,7 @@ sb_time(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(0, SCMP_CMP_EQ, 0)); #else return 0; -#endif +#endif /* defined(__NR_time) */ } /** @@ -356,7 +337,7 @@ sb_accept4(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (rc) { return rc; } -#endif +#endif /* defined(__i386__) */ rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(accept4), SCMP_CMP_MASKED(3, SOCK_CLOEXEC|SOCK_NONBLOCK, 0)); @@ -429,7 +410,7 @@ sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } -#endif +#endif /* defined(__NR_mmap2) */ #ifdef HAVE_GNU_LIBC_VERSION_H #ifdef HAVE_GNU_GET_LIBC_VERSION @@ -457,9 +438,9 @@ libc_uses_openat_for_everything(void) return 1; else return 0; -#else +#else /* !(defined(CHECK_LIBC_VERSION)) */ return 0; -#endif +#endif /* defined(CHECK_LIBC_VERSION) */ } /** Allow a single file to be opened. If <b>use_openat</b> is true, @@ -540,7 +521,7 @@ sb_chmod(scmp_filter_ctx ctx, sandbox_cfg_t *filter) rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chmod), SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + log_err(LD_BUG,"(Sandbox) failed to add chmod syscall, received " "libseccomp error %d", rc); return rc; } @@ -565,7 +546,7 @@ sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter) rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chown), SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + log_err(LD_BUG,"(Sandbox) failed to add chown syscall, received " "libseccomp error %d", rc); return rc; } @@ -744,6 +725,25 @@ sb_socketpair(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#ifdef HAVE_KIST_SUPPORT + +#include <linux/sockios.h> + +static int +sb_ioctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), + SCMP_CMP(1, SCMP_CMP_EQ, SIOCOUTQNSD)); + if (rc) + return rc; + return 0; +} + +#endif /* defined(HAVE_KIST_SUPPORT) */ + /** * Function responsible for setting up the setsockopt syscall for * the seccomp filter sandbox. @@ -784,7 +784,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUFFORCE)); if (rc) return rc; -#endif +#endif /* defined(HAVE_SYSTEMD) */ #ifdef IP_TRANSPARENT rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), @@ -792,7 +792,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, IP_TRANSPARENT)); if (rc) return rc; -#endif +#endif /* defined(IP_TRANSPARENT) */ #ifdef IPV6_V6ONLY rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), @@ -800,7 +800,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, IPV6_V6ONLY)); if (rc) return rc; -#endif +#endif /* defined(IPV6_V6ONLY) */ return 0; } @@ -833,7 +833,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUF)); if (rc) return rc; -#endif +#endif /* defined(HAVE_SYSTEMD) */ #ifdef HAVE_LINUX_NETFILTER_IPV4_H rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), @@ -841,7 +841,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, SO_ORIGINAL_DST)); if (rc) return rc; -#endif +#endif /* defined(HAVE_LINUX_NETFILTER_IPV4_H) */ #ifdef HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), @@ -849,7 +849,16 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(2, SCMP_CMP_EQ, IP6T_SO_ORIGINAL_DST)); if (rc) return rc; -#endif +#endif /* defined(HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H) */ + +#ifdef HAVE_KIST_SUPPORT +#include <netinet/tcp.h> + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_TCP), + SCMP_CMP(2, SCMP_CMP_EQ, TCP_INFO)); + if (rc) + return rc; +#endif /* defined(HAVE_KIST_SUPPORT) */ return 0; } @@ -889,7 +898,7 @@ sb_fcntl64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } -#endif +#endif /* defined(__NR_fcntl64) */ /** * Function responsible for setting up the epoll_ctl syscall for @@ -1086,8 +1095,8 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(stat64), SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); if (rc != 0) { - log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " - "libseccomp error %d", rc); + log_err(LD_BUG,"(Sandbox) failed to add stat64 syscall, received " + "libseccomp error %d", rc); return rc; } } @@ -1095,7 +1104,7 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } -#endif +#endif /* defined(__NR_stat64) */ static int sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter) @@ -1107,7 +1116,7 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter) SCMP_CMP(1, SCMP_CMP_EQ, 0)); #else return 0; -#endif +#endif /* defined(__NR_kill) */ } /** @@ -1117,9 +1126,6 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter) static sandbox_filter_func_t filter_func[] = { sb_rt_sigaction, sb_rt_sigprocmask, -#if 0 - sb_execve, -#endif sb_time, sb_accept4, #ifdef __NR_mmap2 @@ -1148,6 +1154,9 @@ static sandbox_filter_func_t filter_func[] = { sb_setsockopt, sb_getsockopt, sb_socketpair, +#ifdef HAVE_KIST_SUPPORT + sb_ioctl, +#endif sb_kill }; @@ -1374,10 +1383,6 @@ sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_stat, file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1391,10 +1396,6 @@ sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(open), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1408,10 +1409,6 @@ sandbox_cfg_allow_chmod_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(chmod), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1425,10 +1422,6 @@ sandbox_cfg_allow_chown_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(chown), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1443,11 +1436,6 @@ sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2) elem = new_element2(SCMP_SYS(rename), file1, file2); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } - elem->next = *cfg; *cfg = elem; @@ -1460,28 +1448,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) sandbox_cfg_t *elem = NULL; elem = new_element(SCMP_SYS(openat), file); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } - - elem->next = *cfg; - *cfg = elem; - - return 0; -} - -#if 0 -int -sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) -{ - sandbox_cfg_t *elem = NULL; - - elem = new_element(SCMP_SYS(execve), com); - if (!elem) { - log_err(LD_BUG,"(Sandbox) failed to register parameter!"); - return -1; - } elem->next = *cfg; *cfg = elem; @@ -1489,8 +1455,6 @@ sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) return 0; } -#endif - /** Cache entry for getaddrinfo results; used when sandboxing is implemented * so that we can consult the cache when the sandbox prevents us from doing * getaddrinfo. @@ -1521,8 +1485,12 @@ cached_getaddrinfo_items_eq(const cached_getaddrinfo_item_t *a, return (a->family == b->family) && 0 == strcmp(a->name, b->name); } +#define cached_getaddrinfo_item_free(item) \ + FREE_AND_NULL(cached_getaddrinfo_item_t, \ + cached_getaddrinfo_item_free_, (item)) + static void -cached_getaddrinfo_item_free(cached_getaddrinfo_item_t *item) +cached_getaddrinfo_item_free_(cached_getaddrinfo_item_t *item) { if (item == NULL) return; @@ -1614,7 +1582,7 @@ sandbox_getaddrinfo(const char *name, const char *servname, return err; } - /* Otherwise, the sanbox is on. If we have an item, yield its cached + /* Otherwise, the sandbox is on. If we have an item, yield its cached result. */ if (item) { *res = item->res; @@ -1747,7 +1715,9 @@ install_syscall_filter(sandbox_cfg_t* cfg) // loading the seccomp2 filter if ((rc = seccomp_load(ctx))) { - log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)!", rc, + log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)! " + "Are you sure that your kernel has seccomp2 support? The " + "sandbox won't work without it.", rc, strerror(-rc)); goto end; } @@ -1815,7 +1785,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context) /* Clean up the top stack frame so we get the real function * name for the most recently failing function. */ clean_backtrace(syscall_cb_buf, depth, ctx); -#endif +#endif /* defined(USE_BACKTRACE) */ syscall_name = get_syscall_name(syscall); @@ -1831,7 +1801,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context) #endif #if defined(DEBUGGING_CLOSE) - _exit(1); + _exit(1); // exit ok: programming error has led to sandbox failure. #endif // DEBUGGING_CLOSE } @@ -1889,7 +1859,7 @@ register_cfg(sandbox_cfg_t* cfg) return 0; } -#endif // USE_LIBSECCOMP +#endif /* defined(USE_LIBSECCOMP) */ #ifdef USE_LIBSECCOMP /** @@ -1919,7 +1889,7 @@ sandbox_is_active(void) { return sandbox_active != 0; } -#endif // USE_LIBSECCOMP +#endif /* defined(USE_LIBSECCOMP) */ sandbox_cfg_t* sandbox_cfg_new(void) @@ -1947,7 +1917,7 @@ sandbox_init(sandbox_cfg_t *cfg) "Currently, sandboxing is only implemented on Linux. The feature " "is disabled on your platform."); return 0; -#endif +#endif /* defined(USE_LIBSECCOMP) || ... */ } #ifndef USE_LIBSECCOMP @@ -1965,15 +1935,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) return 0; } -#if 0 -int -sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) -{ - (void)cfg; (void)com; - return 0; -} -#endif - int sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) { @@ -2012,5 +1973,5 @@ void sandbox_disable_getaddrinfo_cache(void) { } -#endif +#endif /* !defined(USE_LIBSECCOMP) */ diff --git a/src/common/sandbox.h b/src/common/sandbox.h index c5963e3119..d0f85570f4 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,7 +23,7 @@ */ #define SYS_SECCOMP 1 -#endif +#endif /* !defined(SYS_SECCOMP) */ #if defined(HAVE_SECCOMP_H) && defined(__linux__) #define USE_LIBSECCOMP @@ -101,7 +101,7 @@ typedef struct { sandbox_cfg_t *filter_dynamic; } sandbox_t; -#endif // USE_LIBSECCOMP +#endif /* defined(USE_LIBSECCOMP) */ #ifdef USE_LIBSECCOMP /** Pre-calls getaddrinfo in order to pre-record result. */ @@ -114,7 +114,7 @@ int sandbox_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void sandbox_freeaddrinfo(struct addrinfo *addrinfo); void sandbox_free_getaddrinfo_cache(void); -#else +#else /* !(defined(USE_LIBSECCOMP)) */ #define sandbox_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) #define sandbox_add_addrinfo(name) \ @@ -122,16 +122,16 @@ void sandbox_free_getaddrinfo_cache(void); #define sandbox_freeaddrinfo(addrinfo) \ freeaddrinfo((addrinfo)) #define sandbox_free_getaddrinfo_cache() -#endif +#endif /* defined(USE_LIBSECCOMP) */ #ifdef USE_LIBSECCOMP /** Returns a registered protected string used with the sandbox, given that * it matches the parameter. */ const char* sandbox_intern_string(const char *param); -#else +#else /* !(defined(USE_LIBSECCOMP)) */ #define sandbox_intern_string(s) (s) -#endif +#endif /* defined(USE_LIBSECCOMP) */ /** Creates an empty sandbox configuration file.*/ sandbox_cfg_t * sandbox_cfg_new(void); @@ -156,14 +156,6 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2); */ int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file); -#if 0 -/** - * Function used to add a execve allowed filename to a supplied configuration. - * The (char*) specifies the path to the allowed file; that pointer is stolen. - */ -int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com); -#endif - /** * Function used to add a stat/stat64 allowed filename to a configuration. * The (char*) specifies the path to the allowed file; that pointer is stolen. @@ -178,5 +170,5 @@ int sandbox_is_active(void); void sandbox_disable_getaddrinfo_cache(void); -#endif /* SANDBOX_H_ */ +#endif /* !defined(SANDBOX_H_) */ diff --git a/src/common/storagedir.c b/src/common/storagedir.c new file mode 100644 index 0000000000..e2c7b4bb87 --- /dev/null +++ b/src/common/storagedir.c @@ -0,0 +1,586 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "container.h" +#include "compat.h" +#include "confline.h" +#include "memarea.h" +#include "sandbox.h" +#include "storagedir.h" +#include "torlog.h" +#include "util.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#define FNAME_MIN_NUM 1000 + +/** A storage_dir_t represents a directory full of similar cached + * files. Filenames are decimal integers. Files can be cleaned as needed + * to limit total disk usage. */ +struct storage_dir_t { + /** Directory holding the files for this storagedir. */ + char *directory; + /** Either NULL, or a directory listing of the directory (as a smartlist + * of strings */ + smartlist_t *contents; + /** The largest number of non-temporary files we'll place in the + * directory. */ + int max_files; + /** If true, then 'usage' has been computed. */ + int usage_known; + /** The total number of bytes used in this directory */ + uint64_t usage; +}; + +/** Create or open a new storage directory at <b>dirname</b>, with + * capacity for up to <b>max_files</b> files. + */ +storage_dir_t * +storage_dir_new(const char *dirname, int max_files) +{ + if (check_private_dir(dirname, CPD_CREATE, NULL) < 0) + return NULL; + + storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t)); + d->directory = tor_strdup(dirname); + d->max_files = max_files; + return d; +} + +/** + * Drop all in-RAM storage for <b>d</b>. Does not delete any files. + */ +void +storage_dir_free_(storage_dir_t *d) +{ + if (d == NULL) + return; + tor_free(d->directory); + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + tor_free(d); +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that <b>d</b> will need. + * + * The presence of this function is why we need an upper limit on the + * number of files in a storage_dir_t: we need to approve file operations + * one by one. + */ +int +storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg) +{ + int problems = 0; + int idx; + for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) { + char *path = NULL, *tmppath = NULL; + tor_asprintf(&path, "%s/%d", d->directory, idx); + tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx); + + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_rename(cfg, + tor_strdup(tmppath), tor_strdup(path)); + + tor_free(path); + tor_free(tmppath); + } + + return problems ? -1 : 0; +} + +/** + * Remove all files in <b>d</b> whose names end with ".tmp". + * + * Requires that the contents field of <b>d</b> is set. + */ +static void +storage_dir_clean_tmpfiles(storage_dir_t *d) +{ + if (!d->contents) + return; + SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) { + if (strcmpend(fname, ".tmp")) + continue; + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + if (unlink(sandbox_intern_string(path))) { + log_warn(LD_FS, "Unable to unlink %s while cleaning " + "temporary files: %s", escaped(path), strerror(errno)); + tor_free(path); + continue; + } + tor_free(path); + SMARTLIST_DEL_CURRENT(d->contents, fname); + tor_free(fname); + } SMARTLIST_FOREACH_END(fname); + + d->usage_known = 0; +} + +/** + * Re-scan the directory <b>d</b> to learn its contents. + */ +static int +storage_dir_rescan(storage_dir_t *d) +{ + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + d->usage = 0; + d->usage_known = 0; + if (NULL == (d->contents = tor_listdir(d->directory))) { + return -1; + } + storage_dir_clean_tmpfiles(d); + return 0; +} + +/** + * Return a smartlist containing the filenames within <b>d</b>. + */ +const smartlist_t * +storage_dir_list(storage_dir_t *d) +{ + if (! d->contents) + storage_dir_rescan(d); + return d->contents; +} + +/** + * Return the total number of bytes used for storage in <b>d</b>. + */ +uint64_t +storage_dir_get_usage(storage_dir_t *d) +{ + if (d->usage_known) + return d->usage; + + uint64_t total = 0; + SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) { + char *path = NULL; + struct stat st; + tor_asprintf(&path, "%s/%s", d->directory, cp); + if (stat(sandbox_intern_string(path), &st) == 0) { + total += st.st_size; + } + tor_free(path); + } SMARTLIST_FOREACH_END(cp); + + d->usage = total; + d->usage_known = 1; + return d->usage; +} + +/** Mmap a specified file within <b>d</b>. + * + * On failure, return NULL and set errno as for tor_mmap_file(). */ +tor_mmap_t * +storage_dir_map(storage_dir_t *d, const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + tor_mmap_t *result = tor_mmap_file(path); + int errval = errno; + tor_free(path); + if (result == NULL) + errno = errval; + return result; +} + +/** Read a file within <b>d</b> into a newly allocated buffer. Set + * *<b>sz_out</b> to its size. */ +uint8_t * +storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out) +{ + const int flags = bin ? RFTS_BIN : 0; + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + struct stat st; + char *contents = read_file_to_str(path, flags, &st); + if (contents && sz_out) { + // it fits in RAM, so we know its size is less than SIZE_MAX +#if UINT64_MAX > SIZE_MAX + tor_assert((uint64_t)st.st_size <= SIZE_MAX); +#endif + *sz_out = (size_t) st.st_size; + } + + tor_free(path); + return (uint8_t *) contents; +} + +/** Helper: Find an unused filename within the directory */ +static char * +find_unused_fname(storage_dir_t *d) +{ + if (!d->contents) { + if (storage_dir_rescan(d) < 0) + return NULL; + } + + char buf[16]; + int i; + /* Yuck; this is quadratic. Fortunately, that shouldn't matter much, + * since disk writes are more expensive by a lot. */ + for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) { + tor_snprintf(buf, sizeof(buf), "%d", i); + if (!smartlist_contains_string(d->contents, buf)) { + return tor_strdup(buf); + } + } + return NULL; +} + +/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of + * sized_chunk_t rather than a single byte array. */ +static int +storage_dir_save_chunks_to_file(storage_dir_t *d, + const smartlist_t *chunks, + int binary, + char **fname_out) +{ + uint64_t total_length = 0; + char *fname = find_unused_fname(d); + if (!fname) + return -1; + + SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch, + total_length += ch->len); + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + + int r = write_chunks_to_file(path, chunks, binary, 0); + if (r == 0) { + if (d->usage_known) + d->usage += total_length; + if (fname_out) { + *fname_out = tor_strdup(fname); + } + if (d->contents) + smartlist_add(d->contents, tor_strdup(fname)); + } + tor_free(fname); + tor_free(path); + return r; +} + +/** Try to write the <b>length</b> bytes at <b>data</b> into a new file + * in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a + * newly allocated string containing the filename. On failure, return + * -1. */ +int +storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out) +{ + smartlist_t *chunks = smartlist_new(); + sized_chunk_t chunk = { (const char *)data, length }; + smartlist_add(chunks, &chunk); + int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out); + smartlist_free(chunks); + return r; +} + +/** + * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string + * <b>str</b>. + */ +int +storage_dir_save_string_to_file(storage_dir_t *d, + const char *str, + int binary, + char **fname_out) +{ + return storage_dir_save_bytes_to_file(d, + (const uint8_t*)str, strlen(str), binary, fname_out); +} + +/** + * As storage_dir_save_bytes_to_file, but associates the data with the + * key-value pairs in <b>labels</b>. Files stored in this format can be + * recovered with storage_dir_map_labeled() or storage_dir_read_labeled(). + */ +int +storage_dir_save_labeled_to_file(storage_dir_t *d, + const config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out) +{ + /* + * The storage format is to prefix the data with the key-value pairs in + * <b>labels</b>, and a single NUL separator. But code outside this module + * MUST NOT rely on that format. + */ + + smartlist_t *chunks = smartlist_new(); + memarea_t *area = memarea_new(); + const config_line_t *line; + for (line = labels; line; line = line->next) { + sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t)); + sz->len = strlen(line->key) + 1 + strlen(line->value) + 1; + const size_t allocated = sz->len + 1; + char *bytes = memarea_alloc(area, allocated); + tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value); + sz->bytes = bytes; + smartlist_add(chunks, sz); + } + + sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t)); + nul->len = 1; + nul->bytes = "\0"; + smartlist_add(chunks, nul); + + sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t)); + datachunk->bytes = (const char *)data; + datachunk->len = length; + smartlist_add(chunks, datachunk); + + int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out); + smartlist_free(chunks); + memarea_drop_all(area); + return r; +} + +/** + * Map a file that was created with storage_dir_save_labeled_to_file(). On + * failure, return NULL. On success, write a set of newly allocated labels + * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and + * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t + * object whose contents should not be used -- it needs to be kept around, + * though, for as long as <b>data_out</b> is going to be valid. + * + * On failure, set errno as for tor_mmap_file() if the file was missing or + * empty, and set errno to EINVAL if the file was not in the labeled + * format expected. + */ +tor_mmap_t * +storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + const uint8_t **data_out, + size_t *sz_out) +{ + tor_mmap_t *m = storage_dir_map(dir, fname); + int errval; + if (! m) { + errval = errno; + goto err; + } + const char *nulp = memchr(m->data, '\0', m->size); + if (! nulp) { + errval = EINVAL; + goto err; + } + if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) { + errval = EINVAL; + goto err; + } + size_t offset = nulp - m->data + 1; + tor_assert(offset <= m->size); + *data_out = (const uint8_t *)(m->data + offset); + *sz_out = m->size - offset; + + return m; + err: + tor_munmap_file(m); + errno = errval; + return NULL; +} + +/** As storage_dir_map_labeled, but return a new byte array containing the + * data. */ +uint8_t * +storage_dir_read_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + size_t *sz_out) +{ + const uint8_t *data = NULL; + tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out, + &data, sz_out); + if (m == NULL) + return NULL; + uint8_t *result = tor_memdup(data, *sz_out); + tor_munmap_file(m); + return result; +} + +/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>. + * This function is a no-op if <b>d->usage_known</b> is 0. */ +static void +storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size) +{ + if (d->usage_known) { + if (! BUG(d->usage < removed_file_size)) { + /* This bug can also be triggered if an external process resized a file + * between the call to storage_dir_get_usage() that last checked + * actual usage (rather than relaying on cached usage), and the call to + * this function. */ + d->usage -= removed_file_size; + } else { + /* If we underflowed the cached directory size, re-check the sizes of all + * the files in the directory. This makes storage_dir_shrink() quadratic, + * but only if a process is continually changing file sizes in the + * storage directory (in which case, we have bigger issues). + * + * We can't just reset usage_known, because storage_dir_shrink() relies + * on knowing the usage. */ + storage_dir_rescan(d); + (void)storage_dir_get_usage(d); + } + } +} + +/** + * Remove the file called <b>fname</b> from <b>d</b>. + */ +void +storage_dir_remove_file(storage_dir_t *d, + const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + const char *ipath = sandbox_intern_string(path); + + uint64_t size = 0; + if (d->usage_known) { + struct stat st; + if (stat(ipath, &st) == 0) { + size = st.st_size; + } + } + if (unlink(ipath) == 0) { + storage_dir_reduce_usage(d, size); + } else { + log_warn(LD_FS, "Unable to unlink %s while removing file: %s", + escaped(path), strerror(errno)); + tor_free(path); + return; + } + if (d->contents) { + smartlist_string_remove(d->contents, fname); + } + + tor_free(path); +} + +/** Helper type: used to sort the members of storage directory by mtime. */ +typedef struct shrinking_dir_entry_t { + time_t mtime; + uint64_t size; + char *path; +} shrinking_dir_entry_t; + +/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */ +static int +shrinking_dir_entry_compare(const void *a_, const void *b_) +{ + const shrinking_dir_entry_t *a = a_; + const shrinking_dir_entry_t *b = b_; + + if (a->mtime < b->mtime) + return -1; + else if (a->mtime > b->mtime) + return 1; + else + return 0; +} + +/** + * Try to free space by removing the oldest files in <b>d</b>. Delete + * until no more than <b>target_size</b> bytes are left, and at least + * <b>min_to_remove</b> files have been removed... or until there is + * nothing left to remove. + * + * Return 0 on success; -1 on failure. + */ +int +storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove) +{ + if (d->usage_known && d->usage <= target_size && !min_to_remove) { + /* Already small enough. */ + return 0; + } + + if (storage_dir_rescan(d) < 0) + return -1; + + const uint64_t orig_usage = storage_dir_get_usage(d); + if (orig_usage <= target_size && !min_to_remove) { + /* Okay, small enough after rescan! */ + return 0; + } + + const int n = smartlist_len(d->contents); + shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t)); + SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) { + shrinking_dir_entry_t *ent = &ents[fname_sl_idx]; + struct stat st; + tor_asprintf(&ent->path, "%s/%s", d->directory, fname); + if (stat(sandbox_intern_string(ent->path), &st) == 0) { + ent->mtime = st.st_mtime; + ent->size = st.st_size; + } + } SMARTLIST_FOREACH_END(fname); + + qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare); + + int idx = 0; + while ((d->usage > target_size || min_to_remove > 0) && idx < n) { + if (unlink(sandbox_intern_string(ents[idx].path)) == 0) { + storage_dir_reduce_usage(d, ents[idx].size); + --min_to_remove; + } + ++idx; + } + + for (idx = 0; idx < n; ++idx) { + tor_free(ents[idx].path); + } + tor_free(ents); + + storage_dir_rescan(d); + + return 0; +} + +/** Remove all files in <b>d</b>. */ +int +storage_dir_remove_all(storage_dir_t *d) +{ + return storage_dir_shrink(d, 0, d->max_files); +} + +/** + * Return the largest number of non-temporary files we're willing to + * store in <b>d</b>. + */ +int +storage_dir_get_max_files(storage_dir_t *d) +{ + return d->max_files; +} + diff --git a/src/common/storagedir.h b/src/common/storagedir.h new file mode 100644 index 0000000000..d99bd7ec52 --- /dev/null +++ b/src/common/storagedir.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_STORAGEDIR_H +#define TOR_STORAGEDIR_H + +typedef struct storage_dir_t storage_dir_t; +struct config_line_t; +struct sandbox_cfg_elem; + +storage_dir_t * storage_dir_new(const char *dirname, int n_files); +void storage_dir_free_(storage_dir_t *d); +#define storage_dir_free(d) \ + FREE_AND_NULL(storage_dir_t, storage_dir_free_, (d)) + +int storage_dir_register_with_sandbox(storage_dir_t *d, + struct sandbox_cfg_elem **cfg); +const smartlist_t *storage_dir_list(storage_dir_t *d); +uint64_t storage_dir_get_usage(storage_dir_t *d); +tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname); +uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin, + size_t *sz_out); +int storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out); +int storage_dir_save_string_to_file(storage_dir_t *d, + const char *data, + int binary, + char **fname_out); +int storage_dir_save_labeled_to_file(storage_dir_t *d, + const struct config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out); +tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + struct config_line_t **labels_out, + const uint8_t **data_out, + size_t *size_out); +uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname, + struct config_line_t **labels_out, + size_t *sz_out); +void storage_dir_remove_file(storage_dir_t *d, + const char *fname); +int storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove); +int storage_dir_remove_all(storage_dir_t *d); +int storage_dir_get_max_files(storage_dir_t *d); + +#endif /* !defined(TOR_STORAGEDIR_H) */ + diff --git a/src/common/testsupport.h b/src/common/testsupport.h index 9ad2ba77e0..a3f2ff91ed 100644 --- a/src/common/testsupport.h +++ b/src/common/testsupport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TESTSUPPORT_H @@ -10,7 +10,7 @@ #else #define STATIC static #define EXTERN(type, name) -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Quick and dirty macros to implement test mocking. * @@ -76,15 +76,15 @@ do { \ func = func ##__real; \ } while (0) -#else +#else /* !(defined(TOR_UNIT_TESTS)) */ #define MOCK_DECL(rv, funcname, arglist) \ rv funcname arglist #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ rv funcname arglist attr #define MOCK_IMPL(rv, funcname, arglist) \ rv funcname arglist -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** @} */ -#endif +#endif /* !defined(TOR_TESTSUPPORT_H) */ diff --git a/src/common/timers.c b/src/common/timers.c index 41b2008ac4..6f6236ed3b 100644 --- a/src/common/timers.c +++ b/src/common/timers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,14 +29,14 @@ #include "orconfig.h" +#define TOR_TIMERS_PRIVATE + #include "compat.h" #include "compat_libevent.h" #include "timers.h" #include "torlog.h" #include "util.h" -#include <event2/event.h> - struct timeout_cb { timer_cb_fn_t cb; void *arg; @@ -51,7 +51,7 @@ struct timeout_cb { #else /* We're not exposing any of the functions outside this file. */ #define TIMEOUT_PUBLIC static -#endif +#endif /* defined(__GNUC__) */ /* We're not using periodic events. */ #define TIMEOUT_DISABLE_INTERVALS /* We always know the global_timeouts object, so we don't need each timeout @@ -61,13 +61,18 @@ struct timeout_cb { #define TIMEOUT_CB_OVERRIDE /* We're going to support timers that are pretty far out in advance. Making * this big can be inefficient, but having a significant number of timers - * above TIMEOUT_MAX can also be super-inefficent. Choosing 5 here sets + * above TIMEOUT_MAX can also be super-inefficient. Choosing 5 here sets * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */ #define WHEEL_NUM 5 +#if SIZEOF_VOID_P == 4 +/* On 32-bit platforms, we want to override wheel_bit, so that timeout.c will + * use 32-bit math. */ +#define WHEEL_BIT 5 +#endif #include "src/ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; -static struct event *global_timer_event = NULL; +static struct mainloop_event_t *global_timer_event = NULL; static monotime_t start_of_time; @@ -145,20 +150,14 @@ libevent_timer_reschedule(void) if (delay > MIN_CHECK_TICKS) delay = MIN_CHECK_TICKS; timeout_to_tv(delay, &d); - event_add(global_timer_event, &d); + mainloop_event_schedule(global_timer_event, &d); } -/** - * Invoked when the libevent timer has expired: see which tor_timer_t events - * have fired, activate their callbacks, and reschedule the libevent timer. - */ -static void -libevent_timer_callback(evutil_socket_t fd, short what, void *arg) +/** Run the callback of every timer that has expired, based on the current + * output of monotime_get(). */ +STATIC void +timers_run_pending(void) { - (void)fd; - (void)what; - (void)arg; - monotime_t now; monotime_get(&now); timer_advance_to_cur_time(&now); @@ -167,6 +166,19 @@ libevent_timer_callback(evutil_socket_t fd, short what, void *arg) while ((t = timeouts_get(global_timeouts))) { t->callback.cb(t, t->callback.arg, &now); } +} + +/** + * Invoked when the libevent timer has expired: see which tor_timer_t events + * have fired, activate their callbacks, and reschedule the libevent timer. + */ +static void +libevent_timer_callback(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + + timers_run_pending(); libevent_timer_reschedule(); } @@ -181,7 +193,7 @@ timers_initialize(void) if (BUG(global_timeouts)) return; // LCOV_EXCL_LINE - timeout_error_t err; + timeout_error_t err = 0; global_timeouts = timeouts_open(0, &err); if (!global_timeouts) { // LCOV_EXCL_START -- this can only fail on malloc failure. @@ -193,9 +205,8 @@ timers_initialize(void) monotime_init(); monotime_get(&start_of_time); - struct event *timer_event; - timer_event = tor_event_new(tor_libevent_get_base(), - -1, 0, libevent_timer_callback, NULL); + mainloop_event_t *timer_event; + timer_event = mainloop_event_new(libevent_timer_callback, NULL); tor_assert(timer_event); global_timer_event = timer_event; @@ -209,7 +220,7 @@ void timers_shutdown(void) { if (global_timer_event) { - tor_event_free(global_timer_event); + mainloop_event_free(global_timer_event); global_timer_event = NULL; } if (global_timeouts) { @@ -235,7 +246,7 @@ timer_new(timer_cb_fn_t cb, void *arg) * scheduled. */ void -timer_free(tor_timer_t *t) +timer_free_(tor_timer_t *t) { if (! t) return; @@ -255,6 +266,20 @@ timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg) } /** + * Set *<b>cb_out</b> (if provided) to this timer's callback function, + * and *<b>arg_out</b> (if provided) to this timer's callback argument. + */ +void +timer_get_cb(const tor_timer_t *t, + timer_cb_fn_t *cb_out, void **arg_out) +{ + if (cb_out) + *cb_out = t->callback.cb; + if (arg_out) + *arg_out = t->callback.arg; +} + +/** * Schedule the timer t to fire at the current time plus a delay of * <b>delay</b> microseconds. All times are relative to monotime_get(). */ diff --git a/src/common/timers.h b/src/common/timers.h index 5f918f8e15..6d27f3e01e 100644 --- a/src/common/timers.h +++ b/src/common/timers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TIMERS_H @@ -13,12 +13,19 @@ typedef void (*timer_cb_fn_t)(tor_timer_t *, void *, const struct monotime_t *); tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg); void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg); +void timer_get_cb(const tor_timer_t *t, + timer_cb_fn_t *cb_out, void **arg_out); void timer_schedule(tor_timer_t *t, const struct timeval *delay); void timer_disable(tor_timer_t *t); -void timer_free(tor_timer_t *t); +void timer_free_(tor_timer_t *t); +#define timer_free(t) FREE_AND_NULL(tor_timer_t, timer_free_, (t)) void timers_initialize(void); void timers_shutdown(void); +#ifdef TOR_TIMERS_PRIVATE +STATIC void timers_run_pending(void); #endif +#endif /* !defined(TOR_TIMERS_H) */ + diff --git a/src/common/token_bucket.c b/src/common/token_bucket.c new file mode 100644 index 0000000000..f2396ec58a --- /dev/null +++ b/src/common/token_bucket.c @@ -0,0 +1,255 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file token_bucket.c + * \brief Functions to use and manipulate token buckets, used for + * rate-limiting on connections and globally. + * + * Tor uses these token buckets to keep track of bandwidth usage, and + * sometimes other things too. + * + * There are two layers of abstraction here: "raw" token buckets, in which all + * the pieces are decoupled, and "read-write" token buckets, which combine all + * the moving parts into one. + * + * Token buckets may become negative. + **/ + +#define TOKEN_BUCKET_PRIVATE + +#include "token_bucket.h" +#include "util_bug.h" + +/** + * Set the <b>rate</b> and <b>burst</b> value in a token_bucket_cfg. + * + * Note that the <b>rate</b> value is in arbitrary units, but those units will + * determine the units of token_bucket_raw_dec(), token_bucket_raw_refill, and + * so on. + */ +void +token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst) +{ + tor_assert_nonfatal(rate > 0); + tor_assert_nonfatal(burst > 0); + if (burst > TOKEN_BUCKET_MAX_BURST) + burst = TOKEN_BUCKET_MAX_BURST; + + cfg->rate = rate; + cfg->burst = burst; +} + +/** + * Initialize a raw token bucket and its associated timestamp to the "full" + * state, according to <b>cfg</b>. + */ +void +token_bucket_raw_reset(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg) +{ + bucket->bucket = cfg->burst; +} + +/** + * Adust a preexisting token bucket to respect the new configuration + * <b>cfg</b>, by decreasing its current level if needed. */ +void +token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg) +{ + bucket->bucket = MIN(bucket->bucket, cfg->burst); +} + +/** + * Given an amount of <b>elapsed</b> time units, and a bucket configuration + * <b>cfg</b>, refill the level of <b>bucket</b> accordingly. Note that the + * units of time in <b>elapsed</b> must correspond to those used to set the + * rate in <b>cfg</b>, or the result will be illogical. + */ +int +token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed) +{ + const int was_empty = (bucket->bucket <= 0); + /* The casts here prevent an underflow. + * + * Note that even if the bucket value is negative, subtracting it from + * "burst" will still produce a correct result. If this result is + * ridiculously high, then the "elapsed > gap / rate" check below + * should catch it. */ + const size_t gap = ((size_t)cfg->burst) - ((size_t)bucket->bucket); + + if (elapsed > gap / cfg->rate) { + bucket->bucket = cfg->burst; + } else { + bucket->bucket += cfg->rate * elapsed; + } + + return was_empty && bucket->bucket > 0; +} + +/** + * Decrement a provided bucket by <b>n</b> units. Note that <b>n</b> + * must be nonnegative. + */ +int +token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n) +{ + if (BUG(n < 0)) + return 0; + const int becomes_empty = bucket->bucket > 0 && n >= bucket->bucket; + bucket->bucket -= n; + return becomes_empty; +} + +/** Convert a rate in bytes per second to a rate in bytes per step */ +STATIC uint32_t +rate_per_sec_to_rate_per_step(uint32_t rate) +{ + /* + The precise calculation we'd want to do is + + (rate / 1000) * to_approximate_msec(TICKS_PER_STEP). But to minimize + rounding error, we do it this way instead, and divide last. + */ + uint64_t units = (uint64_t) rate * TICKS_PER_STEP; + uint32_t val = (uint32_t) + (monotime_coarse_stamp_units_to_approx_msec(units) / 1000); + return val ? val : 1; +} + +/** + * Initialize a token bucket in *<b>bucket</b>, set up to allow <b>rate</b> + * bytes per second, with a maximum burst of <b>burst</b> bytes. The bucket + * is created such that <b>now_ts</b> is the current timestamp. The bucket + * starts out full. + */ +void +token_bucket_rw_init(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst, + uint32_t now_ts) +{ + memset(bucket, 0, sizeof(token_bucket_rw_t)); + token_bucket_rw_adjust(bucket, rate, burst); + token_bucket_rw_reset(bucket, now_ts); +} + +/** + * Change the configured rate (in bytes per second) and burst (in bytes) + * for the token bucket in *<b>bucket</b>. + */ +void +token_bucket_rw_adjust(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst) +{ + token_bucket_cfg_init(&bucket->cfg, + rate_per_sec_to_rate_per_step(rate), + burst); + token_bucket_raw_adjust(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_adjust(&bucket->write_bucket, &bucket->cfg); +} + +/** + * Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. + */ +void +token_bucket_rw_reset(token_bucket_rw_t *bucket, + uint32_t now_ts) +{ + token_bucket_raw_reset(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_reset(&bucket->write_bucket, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; +} + +/** + * Refill <b>bucket</b> as appropriate, given that the current timestamp + * is <b>now_ts</b>. + * + * Return a bitmask containing TB_READ iff read bucket was empty and became + * nonempty, and TB_WRITE iff the write bucket was empty and became nonempty. + */ +int +token_bucket_rw_refill(token_bucket_rw_t *bucket, + uint32_t now_ts) +{ + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); + if (elapsed_ticks > UINT32_MAX-(300*1000)) { + /* Either about 48 days have passed since the last refill, or the + * monotonic clock has somehow moved backwards. (We're looking at you, + * Windows.). We accept up to a 5 minute jump backwards as + * "unremarkable". + */ + return 0; + } + const uint32_t elapsed_steps = elapsed_ticks / TICKS_PER_STEP; + + if (!elapsed_steps) { + /* Note that if less than one whole step elapsed, we don't advance the + * time in last_refilled_at. That's intentional: we want to make sure + * that we add some bytes to it eventually. */ + return 0; + } + + int flags = 0; + if (token_bucket_raw_refill_steps(&bucket->read_bucket, + &bucket->cfg, elapsed_steps)) + flags |= TB_READ; + if (token_bucket_raw_refill_steps(&bucket->write_bucket, + &bucket->cfg, elapsed_steps)) + flags |= TB_WRITE; + + bucket->last_refilled_at_timestamp = now_ts; + return flags; +} + +/** + * Decrement the read token bucket in <b>bucket</b> by <b>n</b> bytes. + * + * Return true if the bucket was nonempty and became empty; return false + * otherwise. + */ +int +token_bucket_rw_dec_read(token_bucket_rw_t *bucket, + ssize_t n) +{ + return token_bucket_raw_dec(&bucket->read_bucket, n); +} + +/** + * Decrement the write token bucket in <b>bucket</b> by <b>n</b> bytes. + * + * Return true if the bucket was nonempty and became empty; return false + * otherwise. + */ +int +token_bucket_rw_dec_write(token_bucket_rw_t *bucket, + ssize_t n) +{ + return token_bucket_raw_dec(&bucket->write_bucket, n); +} + +/** + * As token_bucket_rw_dec_read and token_bucket_rw_dec_write, in a single + * operation. Return a bitmask of TB_READ and TB_WRITE to indicate + * which buckets became empty. + */ +int +token_bucket_rw_dec(token_bucket_rw_t *bucket, + ssize_t n_read, ssize_t n_written) +{ + int flags = 0; + if (token_bucket_rw_dec_read(bucket, n_read)) + flags |= TB_READ; + if (token_bucket_rw_dec_write(bucket, n_written)) + flags |= TB_WRITE; + return flags; +} + diff --git a/src/common/token_bucket.h b/src/common/token_bucket.h new file mode 100644 index 0000000000..0e7832e838 --- /dev/null +++ b/src/common/token_bucket.h @@ -0,0 +1,118 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file token_bucket_rw.h + * \brief Headers for token_bucket_rw.c + **/ + +#ifndef TOR_TOKEN_BUCKET_H +#define TOR_TOKEN_BUCKET_H + +#include "torint.h" +#include "testsupport.h" + +/** Largest allowable burst value for a token buffer. */ +#define TOKEN_BUCKET_MAX_BURST INT32_MAX + +/** A generic token buffer configuration: determines the number of tokens + * added to the bucket in each time unit (the "rate"), and the maximum number + * of tokens in the bucket (the "burst") */ +typedef struct token_bucket_cfg_t { + uint32_t rate; + int32_t burst; +} token_bucket_cfg_t; + +/** A raw token bucket, decoupled from its configuration and timestamp. */ +typedef struct token_bucket_raw_t { + int32_t bucket; +} token_bucket_raw_t; + +void token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst); + +void token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg); + +void token_bucket_raw_reset(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg); + +int token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n); + +int token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed_steps); + +static inline size_t token_bucket_raw_get(const token_bucket_raw_t *bucket); +/** Return the current number of bytes set in a token bucket. */ +static inline size_t +token_bucket_raw_get(const token_bucket_raw_t *bucket) +{ + return bucket->bucket >= 0 ? bucket->bucket : 0; +} + +/** A convenience type containing all the pieces needed for a coupled + * read-bucket and write-bucket that have the same rate limit, and which use + * "timestamp units" (see compat_time.h) for their time. */ +typedef struct token_bucket_rw_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t read_bucket; + token_bucket_raw_t write_bucket; + uint32_t last_refilled_at_timestamp; +} token_bucket_rw_t; + +void token_bucket_rw_init(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst, + uint32_t now_ts); + +void token_bucket_rw_adjust(token_bucket_rw_t *bucket, + uint32_t rate, uint32_t burst); + +void token_bucket_rw_reset(token_bucket_rw_t *bucket, + uint32_t now_ts); + +#define TB_READ 1 +#define TB_WRITE 2 + +int token_bucket_rw_refill(token_bucket_rw_t *bucket, + uint32_t now_ts); + +int token_bucket_rw_dec_read(token_bucket_rw_t *bucket, + ssize_t n); +int token_bucket_rw_dec_write(token_bucket_rw_t *bucket, + ssize_t n); + +int token_bucket_rw_dec(token_bucket_rw_t *bucket, + ssize_t n_read, ssize_t n_written); + +static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket); +static inline size_t +token_bucket_rw_get_read(const token_bucket_rw_t *bucket) +{ + return token_bucket_raw_get(&bucket->read_bucket); +} + +static inline size_t token_bucket_rw_get_write( + const token_bucket_rw_t *bucket); +static inline size_t +token_bucket_rw_get_write(const token_bucket_rw_t *bucket) +{ + return token_bucket_raw_get(&bucket->write_bucket); +} + +#ifdef TOKEN_BUCKET_PRIVATE + +/* To avoid making the rates too small, we consider units of "steps", + * where a "step" is defined as this many timestamp ticks. Keep this + * a power of two if you can. */ +#define TICKS_PER_STEP 16 + +STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate); + +#endif + +#endif /* TOR_TOKEN_BUCKET_H */ + diff --git a/src/common/torgzip.c b/src/common/torgzip.c deleted file mode 100644 index c44399aa74..0000000000 --- a/src/common/torgzip.c +++ /dev/null @@ -1,586 +0,0 @@ -/* Copyright (c) 2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.c - * \brief A simple in-memory gzip implementation. - **/ - -#include "orconfig.h" - -#include <stdlib.h> -#include <stdio.h> -#include <assert.h> -#include <string.h> -#include "torint.h" - -#ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> -#endif - -#include "util.h" -#include "torlog.h" -#include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include <zlib.h> - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ -static inline int -method_bits(compress_method_t method, zlib_compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - -static inline int -get_memlevel(zlib_compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - -/** @{ */ -/* These macros define the maximum allowable compression factor. Anything of - * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to - * have an uncompression factor (uncompressed size:compressed size ratio) of - * any greater than MAX_UNCOMPRESSION_FACTOR. - * - * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to - * be small to limit the attack multiplier, but we also want it to be large - * enough so that no legitimate document --even ones we might invent in the - * future -- ever compresses by a factor of greater than - * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably - * large range of possible values. IMO, anything over 8 is probably safe; IMO - * anything under 50 is probably sufficient. - */ -#define MAX_UNCOMPRESSION_FACTOR 25 -#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) -/** @} */ - -/** Return true if uncompressing an input of size <b>in_size</b> to an input - * of size at least <b>size_out</b> looks like a compression bomb. */ -static int -is_compression_bomb(size_t size_in, size_t size_out) -{ - if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) - return 0; - - return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); -} - -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the method described in <b>method</b>. Store the - * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - */ -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - /* Falls through. */ - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#ifdef OPENBSD - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); - return -1; -} - -/** Given zero or more zlib-compressed or gzip-compressed strings of - * total length - * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated - * buffer, using the method described in <b>method</b>. Store the uncompressed - * string in *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on - * success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a - * failure; otherwise we decompress as much as we can. Warn about truncated - * or corrupt inputs at <b>protocol_warn_level</b>. - */ -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - /* Falls through. */ - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } - return -1; -} - -/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely - * to be compressed or not. If it is, return the likeliest compression method. - * Otherwise, return UNKNOWN_METHOD. - */ -compress_method_t -detect_compression_method(const char *in, size_t in_len) -{ - if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { - return GZIP_METHOD; - } else if (in_len > 2 && (in[0] & 0x0f) == 8 && - (ntohs(get_uint16(in)) % 31) == 0) { - return ZLIB_METHOD; - } else { - return UNKNOWN_METHOD; - } -} - -/** Internal state for an incremental zlib compression/decompression. The - * body of this struct is not exposed. */ -struct tor_zlib_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ - - /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; -}; - -/** Construct and return a tor_zlib_state_t object using <b>method</b>. If - * <b>compress</b>, it's for compression; otherwise it's for - * decompression. */ -tor_zlib_state_t * -tor_zlib_new(int compress_, compress_method_t method, - zlib_compression_level_t compression_level) -{ - tor_zlib_state_t *out; - int bits, memlevel; - - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } - - out = tor_malloc_zero(sizeof(tor_zlib_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - - total_zlib_allocation += out->allocation; - - return out; - - err: - tor_free(out); - return NULL; -} - -/** Compress/decompress some bytes using <b>state</b>. Read up to - * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes - * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, - * we've reached the end of the input. - * - * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. - * Return TOR_ZLIB_OK if we're processed everything from the input. - * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>. - * Return TOR_ZLIB_ERR if the stream is corrupt. - */ -tor_zlib_output_t -tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) -{ - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; - - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } - - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); - - *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; - - if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_ZLIB_ERR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_ZLIB_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_ZLIB_OK; - return TOR_ZLIB_BUF_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_ZLIB_BUF_FULL; - return TOR_ZLIB_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_ZLIB_ERR; - } -} - -/** Deallocate <b>state</b>. */ -void -tor_zlib_free(tor_zlib_state_t *state) -{ - if (!state) - return; - - total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); - - tor_free(state); -} - -/** Return an approximate number of bytes used in RAM to hold a state with - * window bits <b>windowBits</b> and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - -/** Return the approximate number of bytes allocated for <b>state</b>. */ -size_t -tor_zlib_state_size(const tor_zlib_state_t *state) -{ - return state->allocation; -} - -/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; -} - diff --git a/src/common/torgzip.h b/src/common/torgzip.h deleted file mode 100644 index 00f62dcb45..0000000000 --- a/src/common/torgzip.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.h - * \brief Headers for torgzip.h - **/ - -#ifndef TOR_TORGZIP_H -#define TOR_TORGZIP_H - -/** Enumeration of what kind of compression to use. Only ZLIB_METHOD is - * guaranteed to be supported by the compress/uncompress functions here; - * GZIP_METHOD may be supported if we built against zlib version 1.2 or later - * and is_gzip_supported() returns true. */ -typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 -} compress_method_t; - -/** - * Enumeration to define tradeoffs between memory usage and compression level. - * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most - * memory. - **/ -typedef enum { - HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} zlib_compression_level_t; - -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - -int is_gzip_supported(void); - -const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - -compress_method_t detect_compression_method(const char *in, size_t in_len); - -/** Return values from tor_zlib_process; see that function's documentation for - * details. */ -typedef enum { - TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR -} tor_zlib_output_t; -/** Internal state for an incremental zlib compression/decompression. */ -typedef struct tor_zlib_state_t tor_zlib_state_t; -tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - zlib_compression_level_t level); - -tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_zlib_free(tor_zlib_state_t *state); - -size_t tor_zlib_state_size(const tor_zlib_state_t *state); -size_t tor_zlib_get_total_allocation(void); - -#endif - diff --git a/src/common/torint.h b/src/common/torint.h index 58c30f41a8..fc7818fe2c 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -34,12 +34,14 @@ does the same thing (but doesn't defined __FreeBSD__). */ #include <machine/limits.h> -#endif -#endif +#endif /* !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__) */ +#endif /* defined(HAVE_MACHINE_LIMITS_H) */ #ifdef HAVE_INTTYPES_H #include <inttypes.h> #endif +#include <stdbool.h> + #if (SIZEOF_INT8_T != 0) #define HAVE_INT8_T #endif @@ -80,7 +82,7 @@ typedef signed char int8_t; typedef unsigned char uint8_t; #define HAVE_UINT8_T #endif -#endif +#endif /* (SIZEOF_CHAR == 1) */ #if (SIZEOF_SHORT == 2) #ifndef HAVE_INT16_T @@ -91,7 +93,7 @@ typedef signed short int16_t; typedef unsigned short uint16_t; #define HAVE_UINT16_T #endif -#endif +#endif /* (SIZEOF_SHORT == 2) */ #if (SIZEOF_INT == 2) #ifndef HAVE_INT16_T @@ -129,7 +131,7 @@ typedef unsigned int uint32_t; #ifndef INT32_MIN #define INT32_MIN (-2147483647-1) #endif -#endif +#endif /* (SIZEOF_INT == 2) || ... */ #if (SIZEOF_LONG == 4) #ifndef HAVE_INT32_T @@ -142,7 +144,7 @@ typedef unsigned long uint32_t; #ifndef UINT32_MAX #define UINT32_MAX 0xfffffffful #endif -#endif +#endif /* !defined(HAVE_UINT32_T) */ #elif (SIZEOF_LONG == 8) #ifndef HAVE_INT64_T typedef signed long int64_t; @@ -155,7 +157,7 @@ typedef unsigned long uint64_t; #ifndef UINT64_MAX #define UINT64_MAX 0xfffffffffffffffful #endif -#endif +#endif /* (SIZEOF_LONG == 4) || ... */ #if (SIZEOF_LONG_LONG == 8) #ifndef HAVE_INT64_T @@ -172,7 +174,7 @@ typedef unsigned long long uint64_t; #ifndef INT64_MAX #define INT64_MAX 0x7fffffffffffffffll #endif -#endif +#endif /* (SIZEOF_LONG_LONG == 8) */ #if (SIZEOF___INT64 == 8) #ifndef HAVE_INT64_T @@ -189,7 +191,7 @@ typedef unsigned __int64 uint64_t; #ifndef INT64_MAX #define INT64_MAX 0x7fffffffffffffffi64 #endif -#endif +#endif /* (SIZEOF___INT64 == 8) */ #ifndef INT64_MIN #define INT64_MIN ((- INT64_MAX) - 1) @@ -202,8 +204,8 @@ typedef unsigned __int64 uint64_t; #define SIZE_MAX UINT32_MAX #else #error "Can't define SIZE_MAX" -#endif -#endif +#endif /* SIZEOF_SIZE_T == 8 || ... */ +#endif /* !defined(SIZE_MAX) */ #ifndef HAVE_SSIZE_T #if SIZEOF_SIZE_T == 8 @@ -212,8 +214,8 @@ typedef int64_t ssize_t; typedef int32_t ssize_t; #else #error "Can't define ssize_t." -#endif -#endif +#endif /* SIZEOF_SIZE_T == 8 || ... */ +#endif /* !defined(HAVE_SSIZE_T) */ #if (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8) #ifndef HAVE_INTPTR_T @@ -235,7 +237,7 @@ typedef uint32_t uintptr_t; #endif #else #error "void * is either >8 bytes or <= 2. In either case, I am confused." -#endif +#endif /* (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8) || ... */ #ifndef HAVE_INT8_T #error "Missing type int8_t" @@ -275,8 +277,8 @@ typedef uint32_t uintptr_t; #define LONG_MAX 0x7fffffffffffffffL #else #error "Can't define LONG_MAX" -#endif -#endif +#endif /* (SIZEOF_LONG == 4) || ... */ +#endif /* !defined(LONG_MAX) */ #ifndef INT_MAX #if (SIZEOF_INT == 4) @@ -285,8 +287,8 @@ typedef uint32_t uintptr_t; #define INT_MAX 0x7fffffffffffffffL #else #error "Can't define INT_MAX" -#endif -#endif +#endif /* (SIZEOF_INT == 4) || ... */ +#endif /* !defined(INT_MAX) */ #ifndef UINT_MAX #if (SIZEOF_INT == 2) @@ -297,8 +299,8 @@ typedef uint32_t uintptr_t; #define UINT_MAX 0xffffffffffffffffu #else #error "Can't define UINT_MAX" -#endif -#endif +#endif /* (SIZEOF_INT == 2) || ... */ +#endif /* !defined(UINT_MAX) */ #ifndef SHORT_MAX #if (SIZEOF_SHORT == 2) @@ -307,8 +309,8 @@ typedef uint32_t uintptr_t; #define SHORT_MAX 0x7fffffff #else #error "Can't define SHORT_MAX" -#endif -#endif +#endif /* (SIZEOF_SHORT == 2) || ... */ +#endif /* !defined(SHORT_MAX) */ #ifndef TIME_MAX @@ -320,9 +322,9 @@ typedef uint32_t uintptr_t; #define TIME_MAX ((time_t)INT64_MAX) #else #error "Can't define TIME_MAX" -#endif +#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */ -#endif /* ifndef(TIME_MAX) */ +#endif /* !defined(TIME_MAX) */ #ifndef TIME_MIN @@ -334,9 +336,9 @@ typedef uint32_t uintptr_t; #define TIME_MIN ((time_t)INT64_MIN) #else #error "Can't define TIME_MIN" -#endif +#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */ -#endif /* ifndef(TIME_MIN) */ +#endif /* !defined(TIME_MIN) */ #ifndef SIZE_MAX #if (SIZEOF_SIZE_T == 4) @@ -345,7 +347,17 @@ typedef uint32_t uintptr_t; #define SIZE_MAX UINT64_MAX #else #error "Can't define SIZE_MAX" -#endif +#endif /* (SIZEOF_SIZE_T == 4) || ... */ +#endif /* !defined(SIZE_MAX) */ + +#ifdef _WIN32 +# ifdef _WIN64 +# define TOR_PRIuSZ PRIu64 +# else +# define TOR_PRIuSZ PRIu32 +# endif +#else +# define TOR_PRIuSZ "zu" #endif #ifndef SSIZE_MAX @@ -355,13 +367,13 @@ typedef uint32_t uintptr_t; #define SSIZE_MAX INT64_MAX #else #error "Can't define SSIZE_MAX" -#endif -#endif +#endif /* (SIZEOF_SIZE_T == 4) || ... */ +#endif /* !defined(SSIZE_MAX) */ /** Any ssize_t larger than this amount is likely to be an underflow. */ #define SSIZE_T_CEILING ((ssize_t)(SSIZE_MAX-16)) /** Any size_t larger than this amount is likely to be an underflow. */ #define SIZE_T_CEILING ((size_t)(SSIZE_MAX-16)) -#endif /* __TORINT_H */ +#endif /* !defined(TOR_TORINT_H) */ diff --git a/src/common/torlog.h b/src/common/torlog.h index 20b7d938f0..de389883c0 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,7 +22,7 @@ #error "Your syslog.h thinks high numbers are more important. " \ "We aren't prepared to deal with that." #endif -#else +#else /* !(defined(HAVE_SYSLOG_H)) */ /* Note: Syslog's logging code refers to priorities, with 0 being the most * important. Thus, all our comparisons needed to be reversed when we added * syslog support. @@ -48,7 +48,7 @@ /** Error-level severity: for messages that only appear when something has gone * very wrong, and the Tor process can no longer proceed. */ #define LOG_ERR 3 -#endif +#endif /* defined(HAVE_SYSLOG_H) */ /* Logging domains */ @@ -99,10 +99,14 @@ #define LD_CHANNEL (1u<<21) /** Scheduler */ #define LD_SCHED (1u<<22) +/** Guard nodes */ +#define LD_GUARD (1u<<23) +/** Generation and application of consensus diffs. */ +#define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ -#define LD_DOS (1u<<23) +#define LD_DOS (1u<<25) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 24 +#define N_LOGGING_DOMAINS 26 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ @@ -144,8 +148,14 @@ int add_file_log(const log_severity_list_t *severity, const char *filename, #ifdef HAVE_SYSLOG_H int add_syslog_log(const log_severity_list_t *severity, const char* syslog_identity_tag); -#endif +#endif // HAVE_SYSLOG_H. +#ifdef HAVE_ANDROID_LOG_H +int add_android_log(const log_severity_list_t *severity, + const char *android_identity_tag); +#endif // HAVE_ANDROID_LOG_H. int add_callback_log(const log_severity_list_t *severity, log_callback cb); +typedef void (*pending_callback_callback)(void); +void logs_set_pending_callback_callback(pending_callback_callback cb); void logs_set_domain_logging(int enabled); int get_min_log_level(void); void switch_logs_debug(void); @@ -183,6 +193,10 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, const char *format, ...) CHECK_PRINTF(5,6); +int log_message_is_interesting(int severity, log_domain_mask_t domain); +void tor_log_string(int severity, log_domain_mask_t domain, + const char *function, const char *string); + #if defined(__GNUC__) && __GNUC__ <= 3 /* These are the GCC varidaic macros, so that older versions of GCC don't @@ -211,7 +225,7 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, #define log_err(domain, args...) \ log_fn_(LOG_ERR, domain, __FUNCTION__, args) -#else /* ! defined(__GNUC__) */ +#else /* !(defined(__GNUC__) && __GNUC__ <= 3) */ /* Here are the c99 variadic macros, to work with non-GCC compilers */ @@ -238,7 +252,17 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, #define log_fn_ratelim(ratelim, severity, domain, args,...) \ log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, \ args, ##__VA_ARGS__) -#endif +#endif /* defined(__GNUC__) && __GNUC__ <= 3 */ + +/** This defines log levels that are linked in the Rust log module, rather + * than re-defining these in both Rust and C. + * + * C_RUST_COUPLED src/rust/tor_log LogSeverity, LogDomain + */ +extern const int LOG_WARN_; +extern const int LOG_NOTICE_; +extern const log_domain_mask_t LD_NET_; +extern const log_domain_mask_t LD_GENERAL_; #ifdef LOG_PRIVATE MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, @@ -247,5 +271,5 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, #endif # define TOR_TORLOG_H -#endif +#endif /* !defined(TOR_TORLOG_H) */ diff --git a/src/common/tortls.c b/src/common/tortls.c index 1fbe3c663e..b55511569b 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,6 +17,7 @@ #include "orconfig.h" #define TORTLS_PRIVATE +#define TORTLS_OPENSSL_PRIVATE #include <assert.h> #ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/ @@ -24,6 +25,9 @@ #include <ws2tcpip.h> #endif +#include "crypto.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "compat.h" /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in @@ -31,7 +35,6 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/opensslv.h> -#include "crypto.h" #ifdef OPENSSL_NO_EC #error "We require OpenSSL with ECC support" @@ -55,10 +58,25 @@ ENABLE_GCC_WARNING(redundant-decls) #include "container.h" #include <string.h> +#ifdef OPENSSL_1_1_API +#define X509_get_notBefore_const(cert) \ + X509_get0_notBefore(cert) +#define X509_get_notAfter_const(cert) \ + X509_get0_notAfter(cert) +#ifndef X509_get_notBefore +#define X509_get_notBefore(cert) \ + X509_getm_notBefore(cert) +#endif +#ifndef X509_get_notAfter +#define X509_get_notAfter(cert) \ + X509_getm_notAfter(cert) +#endif +#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert)) +#endif /* Copied from or.h */ #define LEGAL_NICKNAME_CHARACTERS \ @@ -75,7 +93,7 @@ ENABLE_GCC_WARNING(redundant-decls) * SSL3 safely at the same time. */ #define DISABLE_SSL3_HANDSHAKE -#endif +#endif /* OPENSSL_VERSION_NUMBER < OPENSSL_V(1,0,0,'f') */ /* We redefine these so that we can run correctly even if the vendor gives us * a version of OpenSSL that does not match its header files. (Apple: I am @@ -88,6 +106,9 @@ ENABLE_GCC_WARNING(redundant-decls) #define SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION 0x0010 #endif +/** Set to true iff openssl bug 7712 has been detected. */ +static int openssl_bug_7712_is_present = 0; + /** Return values for tor_tls_classify_client_ciphers. * * @{ @@ -136,6 +157,7 @@ static void tor_tls_context_decref(tor_tls_context_t *ctx); static void tor_tls_context_incref(tor_tls_context_t *ctx); static int check_cert_lifetime_internal(int severity, const X509 *cert, + time_t now, int past_tolerance, int future_tolerance); /** Global TLS contexts. We keep them here because nobody else needs @@ -353,8 +375,12 @@ tor_tls_init(void) check_no_tls_errors(); if (!tls_library_is_initialized) { +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL); +#else SSL_library_init(); SSL_load_error_strings(); +#endif #if (SIZEOF_VOID_P >= 8 && \ OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1)) @@ -388,7 +414,7 @@ tor_tls_init(void) "when configuring it) would make ECDH much faster."); } /* LCOV_EXCL_STOP */ -#endif +#endif /* (SIZEOF_VOID_P >= 8 && ... */ tor_tls_allocate_tor_tls_object_ex_data_index(); @@ -442,8 +468,9 @@ tor_x509_name_new(const char *cname) goto error; /* LCOV_EXCL_BR_STOP */ return name; - error: + /* LCOV_EXCL_START : these lines will only execute on out of memory errors*/ + error: X509_NAME_free(name); return NULL; /* LCOV_EXCL_STOP */ @@ -458,11 +485,11 @@ tor_x509_name_new(const char *cname) * Return a certificate on success, NULL on failure. */ MOCK_IMPL(STATIC X509 *, - tor_tls_create_certificate,(crypto_pk_t *rsa, - crypto_pk_t *rsa_sign, - const char *cname, - const char *cname_sign, - unsigned int cert_lifetime)) +tor_tls_create_certificate,(crypto_pk_t *rsa, + crypto_pk_t *rsa_sign, + const char *cname, + const char *cname_sign, + unsigned int cert_lifetime)) { /* OpenSSL generates self-signed certificates with random 64-bit serial * numbers, so let's do that too. */ @@ -482,8 +509,25 @@ MOCK_IMPL(STATIC X509 *, * then we might pick a time where we're about to expire. Lastly, be * sure to start on a day boundary. */ time_t now = time(NULL); - start_time = crypto_rand_time_range(now - cert_lifetime, now) + 2*24*3600; - start_time -= start_time % (24*3600); + /* Our certificate lifetime will be cert_lifetime no matter what, but if we + * start cert_lifetime in the past, we'll have 0 real lifetime. instead we + * start up to (cert_lifetime - min_real_lifetime - start_granularity) in + * the past. */ + const time_t min_real_lifetime = 24*3600; + const time_t start_granularity = 24*3600; + time_t earliest_start_time; + /* Don't actually start in the future! */ + if (cert_lifetime <= min_real_lifetime + start_granularity) { + earliest_start_time = now - 1; + } else { + earliest_start_time = now + min_real_lifetime + start_granularity + - cert_lifetime; + } + start_time = crypto_rand_time_range(earliest_start_time, now); + /* Round the start time back to the start of a day. */ + start_time -= start_time % start_granularity; + + end_time = start_time + cert_lifetime; tor_assert(rsa); tor_assert(cname); @@ -517,12 +561,12 @@ MOCK_IMPL(STATIC X509 *, if (!X509_time_adj(X509_get_notBefore(x509),0,&start_time)) goto error; - end_time = start_time + cert_lifetime; if (!X509_time_adj(X509_get_notAfter(x509),0,&end_time)) goto error; if (!X509_set_pubkey(x509, pkey)) goto error; - if (!X509_sign(x509, sign_pkey, EVP_sha1())) + + if (!X509_sign(x509, sign_pkey, EVP_sha256())) goto error; goto done; @@ -605,6 +649,12 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 ":" #endif +#ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_CCM + TLS1_TXT_DHE_RSA_WITH_AES_256_CCM ":" +#endif +#ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_CCM + TLS1_TXT_DHE_RSA_WITH_AES_128_CCM ":" +#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 ":" #endif @@ -614,8 +664,14 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /* Required */ TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" /* Required */ - TLS1_TXT_DHE_RSA_WITH_AES_128_SHA - ; + TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":" +#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 + TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 ":" +#endif +#ifdef TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 + TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 +#endif + ; /* Note: to set up your own private testing network with link crypto * disabled, set your Tors' cipher list to @@ -637,7 +693,7 @@ static const char CLIENT_CIPHER_LIST[] = /** Free all storage held in <b>cert</b> */ void -tor_x509_cert_free(tor_x509_cert_t *cert) +tor_x509_cert_free_(tor_x509_cert_t *cert) { if (! cert) return; @@ -656,7 +712,7 @@ tor_x509_cert_free(tor_x509_cert_t *cert) * Steals a reference to x509_cert. */ MOCK_IMPL(STATIC tor_x509_cert_t *, - tor_x509_cert_new,(X509 *x509_cert)) +tor_x509_cert_new,(X509 *x509_cert)) { tor_x509_cert_t *cert; EVP_PKEY *pkey; @@ -670,12 +726,7 @@ MOCK_IMPL(STATIC tor_x509_cert_t *, length = i2d_X509(x509_cert, &buf); cert = tor_malloc_zero(sizeof(tor_x509_cert_t)); if (length <= 0 || buf == NULL) { - /* LCOV_EXCL_START for the same reason as the exclusion above */ - tor_free(cert); - log_err(LD_CRYPTO, "Couldn't get length of encoded x509 certificate"); - X509_free(x509_cert); - return NULL; - /* LCOV_EXCL_STOP */ + goto err; } cert->encoded_len = (size_t) length; cert->encoded = tor_malloc(length); @@ -690,13 +741,25 @@ MOCK_IMPL(STATIC tor_x509_cert_t *, if ((pkey = X509_get_pubkey(x509_cert)) && (rsa = EVP_PKEY_get1_RSA(pkey))) { crypto_pk_t *pk = crypto_new_pk_from_rsa_(rsa); - crypto_pk_get_common_digests(pk, &cert->pkey_digests); + if (crypto_pk_get_common_digests(pk, &cert->pkey_digests) < 0) { + crypto_pk_free(pk); + EVP_PKEY_free(pkey); + goto err; + } + cert->pkey_digests_set = 1; crypto_pk_free(pk); EVP_PKEY_free(pkey); } return cert; + err: + /* LCOV_EXCL_START for the same reason as the exclusion above */ + tor_free(cert); + log_err(LD_CRYPTO, "Couldn't wrap encoded X509 certificate."); + X509_free(x509_cert); + return NULL; + /* LCOV_EXCL_STOP */ } /** Return a new copy of <b>cert</b>. */ @@ -800,8 +863,8 @@ tor_tls_context_decref(tor_tls_context_t *ctx) /** Set *<b>link_cert_out</b> and *<b>id_cert_out</b> to the link certificate * and ID certificate that we're currently using for our V3 in-protocol * handshake's certificate chain. If <b>server</b> is true, provide the certs - * that we use in server mode; otherwise, provide the certs that we use in - * client mode. */ + * that we use in server mode (auth, ID); otherwise, provide the certs that we + * use in client mode. (link, ID) */ int tor_tls_get_my_certs(int server, const tor_x509_cert_t **link_cert_out, @@ -831,7 +894,7 @@ tor_tls_get_my_client_auth_key(void) /** * Return a newly allocated copy of the public key that a certificate - * certifies. Return NULL if the cert's key is not RSA. + * certifies. Watch out! This returns NULL if the cert's key is not RSA. */ crypto_pk_t * tor_tls_cert_get_key(tor_x509_cert_t *cert) @@ -888,6 +951,7 @@ int tor_tls_cert_is_valid(int severity, const tor_x509_cert_t *cert, const tor_x509_cert_t *signing_cert, + time_t now, int check_rsa_1024) { check_no_tls_errors(); @@ -907,7 +971,7 @@ tor_tls_cert_is_valid(int severity, /* okay, the signature checked out right. Now let's check the check the * lifetime. */ - if (check_cert_lifetime_internal(severity, cert->cert, + if (check_cert_lifetime_internal(severity, cert->cert, now, 48*60*60, 30*24*60*60) < 0) goto bad; @@ -1052,6 +1116,8 @@ tor_tls_context_init_one(tor_tls_context_t **ppcontext, /** The group we should use for ecdhe when none was selected. */ #define NID_tor_default_ecdhe_group NID_X9_62_prime256v1 +#define RSA_LINK_KEY_BITS 2048 + /** Create a new TLS context for use with Tor TLS handshakes. * <b>identity</b> should be set to the identity key used to sign the * certificate. @@ -1077,7 +1143,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, /* Generate short-term RSA key for use with TLS. */ if (!(rsa = crypto_pk_new())) goto error; - if (crypto_pk_generate_key(rsa)<0) + if (crypto_pk_generate_key_with_bits(rsa, RSA_LINK_KEY_BITS)<0) goto error; if (!is_client) { /* Generate short-term RSA key for use in the in-protocol ("v3") @@ -1120,7 +1186,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, * with existing Tors. */ if (!(result->ctx = SSL_CTX_new(TLSv1_method()))) goto error; -#endif +#endif /* 0 */ /* Tell OpenSSL to use TLS 1.0 or later but not SSL2 or SSL3. */ #ifdef HAVE_TLS_METHOD @@ -1129,7 +1195,8 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, #else if (!(result->ctx = SSL_CTX_new(SSLv23_method()))) goto error; -#endif +#endif /* defined(HAVE_TLS_METHOD) */ + #ifdef HAVE_SSL_CTX_SET_SECURITY_LEVEL /* Level 1 re-enables RSA1024 and DH1024 for compatibility with old tors */ SSL_CTX_set_security_level(result->ctx, 1); @@ -1172,17 +1239,20 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_options(result->ctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); } + + /* Don't actually allow compression; it uses RAM and time, it makes TLS + * vulnerable to CRIME-style attacks, and most of the data we transmit over + * TLS is encrypted (and therefore uncompressible) anyway. */ #ifdef SSL_OP_NO_COMPRESSION SSL_CTX_set_options(result->ctx, SSL_OP_NO_COMPRESSION); #endif #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0) #ifndef OPENSSL_NO_COMP - /* Don't actually allow compression; it uses ram and time, but the data - * we transmit is all encrypted anyway. */ if (result->ctx->comp_methods) result->ctx->comp_methods = NULL; #endif -#endif +#endif /* OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0) */ + #ifdef SSL_MODE_RELEASE_BUFFERS SSL_CTX_set_mode(result->ctx, SSL_MODE_RELEASE_BUFFERS); #endif @@ -1359,7 +1429,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher); return c != NULL; } -#else +#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */ # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) if (m && m->get_cipher_by_char) { @@ -1373,7 +1443,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((c->id & 0xffff) == cipher); return c != NULL; } -# endif +#endif /* defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) */ # ifndef OPENSSL_1_1_API if (m && m->get_cipher && m->num_ciphers) { /* It would seem that some of the "let's-clean-up-openssl" forks have @@ -1389,12 +1459,12 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) } return 0; } -# endif +#endif /* !defined(OPENSSL_1_1_API) */ (void) ssl; (void) m; (void) cipher; return 1; /* No way to search */ -#endif +#endif /* defined(HAVE_SSL_CIPHER_FIND) */ } /** Remove from v2_cipher_list every cipher that we don't support, so that @@ -1522,7 +1592,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl) return CIPHERS_ERR; } ciphers = session->ciphers; -#endif +#endif /* defined(HAVE_SSL_GET_CLIENT_CIPHERS) */ return tor_tls_classify_client_ciphers(ssl, ciphers) >= CIPHERS_V2; } @@ -1653,6 +1723,13 @@ tor_tls_new(int sock, int isServer) SSL_set_tlsext_host_name(result->ssl, fake_hostname); tor_free(fake_hostname); } +#endif /* defined(SSL_set_tlsext_host_name) */ + +#ifdef SSL_CTRL_SET_MAX_PROTO_VERSION + if (openssl_bug_7712_is_present) { + /* We can't actually use TLS 1.3 until this bug is fixed. */ + SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION); + } #endif if (!SSL_set_cipher_list(result->ssl, @@ -1780,7 +1857,7 @@ tor_tls_assert_renegotiation_unblocked(tor_tls_t *tls) tor_assert(0 != (options & SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION)); #else (void) tls; -#endif +#endif /* defined(SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION) && ... */ } /** Return whether this tls initiated the connect (client) or @@ -1796,7 +1873,7 @@ tor_tls_is_server(tor_tls_t *tls) * underlying file descriptor. */ void -tor_tls_free(tor_tls_t *tls) +tor_tls_free_(tor_tls_t *tls) { if (!tls) return; @@ -1943,7 +2020,7 @@ tor_tls_handshake(tor_tls_t *tls) return r; } -/** Perform the final part of the intial TLS handshake on <b>tls</b>. This +/** Perform the final part of the initial TLS handshake on <b>tls</b>. This * should be called for the first handshake only: it determines whether the v1 * or the v2 handshake was used, and adjusts things for the renegotiation * handshake as appropriate. @@ -2097,13 +2174,13 @@ tor_tls_get_own_cert,(tor_tls_t *tls)) /** Warn that a certificate lifetime extends through a certain range. */ static void -log_cert_lifetime(int severity, const X509 *cert, const char *problem) +log_cert_lifetime(int severity, const X509 *cert, const char *problem, + time_t now) { BIO *bio = NULL; BUF_MEM *buf; char *s1=NULL, *s2=NULL; char mytime[33]; - time_t now = time(NULL); struct tm tm; size_t n; @@ -2251,6 +2328,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity_key) */ int tor_tls_check_lifetime(int severity, tor_tls_t *tls, + time_t now, int past_tolerance, int future_tolerance) { X509 *cert; @@ -2259,7 +2337,7 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls, if (!(cert = SSL_get_peer_certificate(tls->ssl))) goto done; - if (check_cert_lifetime_internal(severity, cert, + if (check_cert_lifetime_internal(severity, cert, now, past_tolerance, future_tolerance) < 0) goto done; @@ -2275,30 +2353,48 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls, /** Helper: check whether <b>cert</b> is expired give or take * <b>past_tolerance</b> seconds, or not-yet-valid give or take - * <b>future_tolerance</b> seconds. If it is live, return 0. If it is not - * live, log a message and return -1. */ + * <b>future_tolerance</b> seconds. (Relative to the current time + * <b>now</b>.) If it is live, return 0. If it is not live, log a message + * and return -1. */ static int check_cert_lifetime_internal(int severity, const X509 *cert, + time_t now, int past_tolerance, int future_tolerance) { - time_t now, t; - - now = time(NULL); + time_t t; t = now + future_tolerance; if (X509_cmp_time(X509_get_notBefore_const(cert), &t) > 0) { - log_cert_lifetime(severity, cert, "not yet valid"); + log_cert_lifetime(severity, cert, "not yet valid", now); return -1; } t = now - past_tolerance; if (X509_cmp_time(X509_get_notAfter_const(cert), &t) < 0) { - log_cert_lifetime(severity, cert, "already expired"); + log_cert_lifetime(severity, cert, "already expired", now); return -1; } return 0; } +#ifdef TOR_UNIT_TESTS +/* Testing only: return a new x509 cert with the same contents as <b>inp</b>, + but with the expiration time <b>new_expiration_time</b>, signed with + <b>signing_key</b>. */ +STATIC tor_x509_cert_t * +tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key) +{ + X509 *newc = X509_dup(inp->cert); + X509_time_adj(X509_get_notAfter(newc), 0, &new_expiration_time); + EVP_PKEY *pk = crypto_pk_get_evp_pkey_(signing_key, 1); + tor_assert(X509_sign(newc, pk, EVP_sha256())); + EVP_PKEY_free(pk); + return tor_x509_cert_new(newc); +} +#endif /* defined(TOR_UNIT_TESTS) */ + /** Return the number of bytes available for reading from <b>tls</b>. */ int @@ -2341,10 +2437,10 @@ tor_tls_get_n_raw_bytes(tor_tls_t *tls, size_t *n_read, size_t *n_written) if (BIO_method_type(wbio) == BIO_TYPE_BUFFER && (tmpbio = BIO_next(wbio)) != NULL) wbio = tmpbio; -#else +#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5)) */ if (wbio->method == BIO_f_buffer() && (tmpbio = BIO_next(wbio)) != NULL) wbio = tmpbio; -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) */ w = (unsigned long) BIO_number_written(wbio); /* We are ok with letting these unsigned ints go "negative" here: @@ -2423,7 +2519,7 @@ SSL_get_client_random(SSL *s, uint8_t *out, size_t len) memcpy(out, s->s3->client_random, len); return len; } -#endif +#endif /* !defined(HAVE_SSL_GET_CLIENT_RANDOM) */ #ifndef HAVE_SSL_GET_SERVER_RANDOM static size_t @@ -2436,7 +2532,7 @@ SSL_get_server_random(SSL *s, uint8_t *out, size_t len) memcpy(out, s->s3->server_random, len); return len; } -#endif +#endif /* !defined(HAVE_SSL_GET_SERVER_RANDOM) */ #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY STATIC size_t @@ -2450,7 +2546,7 @@ SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, size_t len) memcpy(out, s->master_key, len); return len; } -#endif +#endif /* !defined(HAVE_SSL_SESSION_GET_MASTER_KEY) */ /** Set the DIGEST256_LEN buffer at <b>secrets_out</b> to the value used in * the v3 handshake to prove that the client knows the TLS secrets for the @@ -2517,6 +2613,62 @@ tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out)) return 0; } +/** Using the RFC5705 key material exporting construction, and the + * provided <b>context</b> (<b>context_len</b> bytes long) and + * <b>label</b> (a NUL-terminated string), compute a 32-byte secret in + * <b>secrets_out</b> that only the parties to this TLS session can + * compute. Return 0 on success; -1 on failure; and -2 on failure + * caused by OpenSSL bug 7712. + */ +MOCK_IMPL(int, +tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out, + const uint8_t *context, + size_t context_len, + const char *label)) +{ + tor_assert(tls); + tor_assert(tls->ssl); + + int r = SSL_export_keying_material(tls->ssl, + secrets_out, DIGEST256_LEN, + label, strlen(label), + context, context_len, 1); + + if (r != 1) { + int severity = openssl_bug_7712_is_present ? LOG_WARN : LOG_DEBUG; + tls_log_errors(tls, severity, LD_NET, "exporting keying material"); + } + +#ifdef TLS1_3_VERSION + if (r != 1 && + strlen(label) > 12 && + SSL_version(tls->ssl) >= TLS1_3_VERSION) { + + if (! openssl_bug_7712_is_present) { + /* We might have run into OpenSSL issue 7712, which caused OpenSSL + * 1.1.1a to not handle long labels. Let's test to see if we have. + */ + r = SSL_export_keying_material(tls->ssl, secrets_out, DIGEST256_LEN, + "short", 5, context, context_len, 1); + if (r == 1) { + /* A short label succeeds, but a long label fails. This was openssl + * issue 7712. */ + openssl_bug_7712_is_present = 1; + log_warn(LD_GENERAL, "Detected OpenSSL bug 7712: disabling TLS 1.3 on " + "future connections. A fix is expected to appear in OpenSSL " + "1.1.1b."); + } + } + if (openssl_bug_7712_is_present) + return -2; + else + return -1; + } +#endif + + return (r == 1) ? 0 : -1; +} + /** Examine the amount of memory used and available for buffers in <b>tls</b>. * Set *<b>rbuf_capacity</b> to the amount of storage allocated for the read * buffer and *<b>rbuf_bytes</b> to the amount actually used. @@ -2537,7 +2689,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls, (void)wbuf_bytes; return -1; -#else +#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)) */ if (tls->ssl->s3->rbuf.buf) *rbuf_capacity = tls->ssl->s3->rbuf.len; else @@ -2549,7 +2701,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls, *rbuf_bytes = tls->ssl->s3->rbuf.left; *wbuf_bytes = tls->ssl->s3->wbuf.left; return 0; -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ } /** Check whether the ECC group requested is supported by the current OpenSSL diff --git a/src/common/tortls.h b/src/common/tortls.h index f018c45c82..7c867bfff2 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_H @@ -11,7 +11,7 @@ * \brief Headers for tortls.c **/ -#include "crypto.h" +#include "crypto_rsa.h" #include "compat_openssl.h" #include "compat.h" #include "testsupport.h" @@ -63,12 +63,17 @@ typedef enum { } tor_tls_state_t; #define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t) +struct x509_st; +struct ssl_st; +struct ssl_ctx_st; +struct ssl_session_st; + /** Holds a SSL_CTX object and related state used to configure TLS * connections. */ typedef struct tor_tls_context_t { int refcnt; - SSL_CTX *ctx; + struct ssl_ctx_st *ctx; tor_x509_cert_t *my_link_cert; tor_x509_cert_t *my_id_cert; tor_x509_cert_t *my_auth_cert; @@ -78,7 +83,7 @@ typedef struct tor_tls_context_t { /** Structure that we use for a single certificate. */ struct tor_x509_cert_t { - X509 *cert; + struct x509_st *cert; uint8_t *encoded; size_t encoded_len; unsigned pkey_digests_set : 1; @@ -92,7 +97,7 @@ struct tor_x509_cert_t { struct tor_tls_t { uint32_t magic; tor_tls_context_t *context; /** A link to the context object for this tls. */ - SSL *ssl; /**< An OpenSSL SSL object. */ + struct ssl_st *ssl; /**< An OpenSSL SSL object. */ int socket; /**< The underlying file descriptor for this TLS connection. */ char *address; /**< An address to log when describing this connection. */ tor_tls_state_bitfield_t state : 3; /**< The current SSL state, @@ -128,35 +133,45 @@ struct tor_tls_t { STATIC int tor_errno_to_tls_error(int e); STATIC int tor_tls_get_error(tor_tls_t *tls, int r, int extra, const char *doing, int severity, int domain); -STATIC tor_tls_t *tor_tls_get_by_ssl(const SSL *ssl); +STATIC tor_tls_t *tor_tls_get_by_ssl(const struct ssl_st *ssl); STATIC void tor_tls_allocate_tor_tls_object_ex_data_index(void); +#ifdef TORTLS_OPENSSL_PRIVATE STATIC int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx); -STATIC int tor_tls_classify_client_ciphers(const SSL *ssl, +STATIC int tor_tls_classify_client_ciphers(const struct ssl_st *ssl, STACK_OF(SSL_CIPHER) *peer_ciphers); -STATIC int tor_tls_client_is_using_v2_ciphers(const SSL *ssl); +#endif +STATIC int tor_tls_client_is_using_v2_ciphers(const struct ssl_st *ssl); MOCK_DECL(STATIC void, try_to_extract_certs_from_tls, - (int severity, tor_tls_t *tls, X509 **cert_out, X509 **id_cert_out)); + (int severity, tor_tls_t *tls, struct x509_st **cert_out, + struct x509_st **id_cert_out)); #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY -STATIC size_t SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, +STATIC size_t SSL_SESSION_get_master_key(struct ssl_session_st *s, + uint8_t *out, size_t len); #endif -STATIC void tor_tls_debug_state_callback(const SSL *ssl, int type, int val); -STATIC void tor_tls_server_info_callback(const SSL *ssl, int type, int val); -STATIC int tor_tls_session_secret_cb(SSL *ssl, void *secret, +STATIC void tor_tls_debug_state_callback(const struct ssl_st *ssl, + int type, int val); +STATIC void tor_tls_server_info_callback(const struct ssl_st *ssl, + int type, int val); +#ifdef TORTLS_OPENSSL_PRIVATE +STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret, int *secret_len, STACK_OF(SSL_CIPHER) *peer_ciphers, CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher, void *arg); STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher); -MOCK_DECL(STATIC X509*, tor_tls_create_certificate,(crypto_pk_t *rsa, +#endif /* defined(TORTLS_OPENSSL_PRIVATE) */ +MOCK_DECL(STATIC struct x509_st *, tor_tls_create_certificate, + (crypto_pk_t *rsa, crypto_pk_t *rsa_sign, const char *cname, const char *cname_sign, unsigned int cert_lifetime)); STATIC tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, unsigned flags, int is_client); -MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,(X509 *x509_cert)); +MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new, + (struct x509_st *x509_cert)); STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext, crypto_pk_t *identity, unsigned int key_lifetime, @@ -172,10 +187,16 @@ extern tor_tls_context_t *client_tls_context; extern uint16_t v2_cipher_list[]; extern uint64_t total_bytes_written_over_tls; extern uint64_t total_bytes_written_by_tls; -#endif -#endif /* endif TORTLS_PRIVATE */ +STATIC tor_x509_cert_t *tor_x509_cert_replace_expiration( + const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key); +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TORTLS_PRIVATE) */ +tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); const char *tor_tls_err_to_string(int err); void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz); @@ -195,14 +216,15 @@ void tor_tls_set_renegotiate_callback(tor_tls_t *tls, void (*cb)(tor_tls_t *, void *arg), void *arg); int tor_tls_is_server(tor_tls_t *tls); -void tor_tls_free(tor_tls_t *tls); +void tor_tls_free_(tor_tls_t *tls); +#define tor_tls_free(tls) FREE_AND_NULL(tor_tls_t, tor_tls_free_, (tls)) int tor_tls_peer_has_cert(tor_tls_t *tls); -tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); MOCK_DECL(tor_x509_cert_t *,tor_tls_get_peer_cert,(tor_tls_t *tls)); MOCK_DECL(tor_x509_cert_t *,tor_tls_get_own_cert,(tor_tls_t *tls)); int tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity); int tor_tls_check_lifetime(int severity, - tor_tls_t *tls, int past_tolerance, + tor_tls_t *tls, time_t now, + int past_tolerance, int future_tolerance); MOCK_DECL(int, tor_tls_read, (tor_tls_t *tls, char *cp, size_t len)); int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n); @@ -228,6 +250,11 @@ int tor_tls_used_v1_handshake(tor_tls_t *tls); int tor_tls_get_num_server_handshakes(tor_tls_t *tls); int tor_tls_server_got_renegotiate(tor_tls_t *tls); MOCK_DECL(int,tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out)); +MOCK_DECL(int,tor_tls_export_key_material,( + tor_tls_t *tls, uint8_t *secrets_out, + const uint8_t *context, + size_t context_len, + const char *label)); /* Log and abort if there are unhandled TLS errors in OpenSSL's error stack. */ @@ -237,7 +264,9 @@ void check_no_tls_errors_(const char *fname, int line); void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err, int severity, int domain, const char *doing); -void tor_x509_cert_free(tor_x509_cert_t *cert); +void tor_x509_cert_free_(tor_x509_cert_t *cert); +#define tor_x509_cert_free(c) \ + FREE_AND_NULL(tor_x509_cert_t, tor_x509_cert_free_, (c)) tor_x509_cert_t *tor_x509_cert_decode(const uint8_t *certificate, size_t certificate_len); void tor_x509_cert_get_der(const tor_x509_cert_t *cert, @@ -256,10 +285,11 @@ MOCK_DECL(int,tor_tls_cert_matches_key,(const tor_tls_t *tls, int tor_tls_cert_is_valid(int severity, const tor_x509_cert_t *cert, const tor_x509_cert_t *signing_cert, + time_t now, int check_rsa_1024); const char *tor_tls_get_ciphersuite_name(tor_tls_t *tls); int evaluate_ecgroup_for_tls(const char *ecgroup); -#endif +#endif /* !defined(TOR_TORTLS_H) */ diff --git a/src/common/util.c b/src/common/util.c index d2cbacde31..dece5877f1 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,7 +16,7 @@ #define UTIL_PRIVATE #include "util.h" #include "torlog.h" -#include "crypto.h" +#include "crypto_digest.h" #include "torint.h" #include "container.h" #include "address.h" @@ -31,11 +31,11 @@ #include <process.h> #include <tchar.h> #include <winbase.h> -#else +#else /* !(defined(_WIN32)) */ #include <dirent.h> #include <pwd.h> #include <grp.h> -#endif +#endif /* defined(_WIN32) */ /* math.h needs this on Linux */ #ifndef _USE_ISOC99_ @@ -79,13 +79,13 @@ #include <malloc/malloc.h> #endif #ifdef HAVE_MALLOC_H -#if !defined(OPENBSD) && !defined(__FreeBSD__) +#if !defined(OpenBSD) && !defined(__FreeBSD__) /* OpenBSD has a malloc.h, but for our purposes, it only exists in order to * scold us for being so stupid as to autodetect its presence. To be fair, * they've done this since 1996, when autoconf was only 5 years old. */ #include <malloc.h> -#endif -#endif +#endif /* !defined(OpenBSD) && !defined(__FreeBSD__) */ +#endif /* defined(HAVE_MALLOC_H) */ #ifdef HAVE_MALLOC_NP_H #include <malloc_np.h> #endif @@ -116,12 +116,12 @@ dmalloc_strndup(file, line, (string), -1, xalloc_b) #else #error "No dmalloc_strdup or equivalent" - #endif +#endif /* defined(HAVE_DMALLOC_STRDUP) || ... */ -#else /* not using dmalloc */ +#else /* !(defined(USE_DMALLOC)) */ #define DMALLOC_FN_ARGS -#endif +#endif /* defined(USE_DMALLOC) */ /** Allocate a chunk of <b>size</b> bytes of memory, and return a pointer to * result. On error, log and terminate the process. (Same as malloc(size), @@ -142,7 +142,7 @@ tor_malloc_(size_t size DMALLOC_PARAMS) if (size==0) { size=1; } -#endif +#endif /* !defined(MALLOC_ZERO_WORKS) */ #ifdef USE_DMALLOC result = dmalloc_malloc(file, line, size, DMALLOC_FUNC_MALLOC, 0, 0); @@ -156,7 +156,7 @@ tor_malloc_(size_t size DMALLOC_PARAMS) /* If these functions die within a worker process, they won't call * spawn_exit, but that's ok, since the parent will run out of memory soon * anyway. */ - exit(1); + exit(1); // exit ok: alloc failed. /* LCOV_EXCL_STOP */ } return result; @@ -187,8 +187,9 @@ tor_malloc_zero_(size_t size DMALLOC_PARAMS) * 0xfffe0001. */ #define SQRT_SIZE_MAX_P1 (((size_t)1) << (sizeof(size_t)*4)) -/** Return non-zero if and only if the product of the arguments is exact. */ -static inline int +/** Return non-zero if and only if the product of the arguments is exact, + * and cannot overflow. */ +int size_mul_check(const size_t x, const size_t y) { /* This first check is equivalent to @@ -202,15 +203,6 @@ size_mul_check(const size_t x, const size_t y) x <= SIZE_MAX / y); } -#ifdef TOR_UNIT_TESTS -/** Exposed for unit tests only */ -int -size_mul_check__(const size_t x, const size_t y) -{ - return size_mul_check(x,y); -} -#endif - /** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill * the memory with zero bytes, and return a pointer to the result. * Log and terminate the process on error. (Same as @@ -241,7 +233,7 @@ tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS) if (size==0) { size=1; } -#endif +#endif /* !defined(MALLOC_ZERO_WORKS) */ #ifdef USE_DMALLOC result = dmalloc_realloc(file, line, ptr, size, DMALLOC_FUNC_REALLOC, 0); @@ -252,7 +244,7 @@ tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS) if (PREDICT_UNLIKELY(result == NULL)) { /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on realloc(). Dying."); - exit(1); + exit(1); // exit ok: alloc failed. /* LCOV_EXCL_STOP */ } return result; @@ -290,7 +282,7 @@ tor_strdup_(const char *s DMALLOC_PARAMS) if (PREDICT_UNLIKELY(duplicate == NULL)) { /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on strdup(). Dying."); - exit(1); + exit(1); // exit ok: alloc failed. /* LCOV_EXCL_STOP */ } return duplicate; @@ -370,16 +362,16 @@ tor_log_mallinfo(int severity) mi.arena, mi.ordblks, mi.smblks, mi.hblks, mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks, mi.keepcost); -#else +#else /* !(defined(HAVE_MALLINFO)) */ (void)severity; -#endif +#endif /* defined(HAVE_MALLINFO) */ #ifdef USE_DMALLOC dmalloc_log_changed(0, /* Since the program started. */ 1, /* Log info about non-freed pointers. */ 0, /* Do not log info about freed pointers. */ 0 /* Do not log individual pointers. */ ); -#endif +#endif /* defined(USE_DMALLOC) */ } ENABLE_GCC_WARNING(aggregate-return) @@ -409,7 +401,7 @@ tor_lround(double d) return (long)rint(d); #else return (long)(d > 0 ? d + 0.5 : ceil(d - 0.5)); -#endif +#endif /* defined(HAVE_LROUND) || ... */ } /** Return the 64-bit integer closest to d. We define this wrapper here so @@ -424,7 +416,7 @@ tor_llround(double d) return (int64_t)rint(d); #else return (int64_t)(d > 0 ? d + 0.5 : ceil(d - 0.5)); -#endif +#endif /* defined(HAVE_LLROUND) || ... */ } /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */ @@ -453,7 +445,7 @@ tor_log2(uint64_t u64) r += 2; } if (u64 >= (U64_LITERAL(1)<<1)) { - u64 >>= 1; + // u64 >>= 1; // not using this any more. r += 1; } return r; @@ -485,7 +477,7 @@ round_to_power_of_2(uint64_t u64) /** Return the lowest x such that x is at least <b>number</b>, and x modulo * <b>divisor</b> == 0. If no such x can be expressed as an unsigned, return - * UINT_MAX */ + * UINT_MAX. Asserts if divisor is zero. */ unsigned round_to_next_multiple_of(unsigned number, unsigned divisor) { @@ -499,7 +491,7 @@ round_to_next_multiple_of(unsigned number, unsigned divisor) /** Return the lowest x such that x is at least <b>number</b>, and x modulo * <b>divisor</b> == 0. If no such x can be expressed as a uint32_t, return - * UINT32_MAX */ + * UINT32_MAX. Asserts if divisor is zero. */ uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) { @@ -514,7 +506,7 @@ round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) /** Return the lowest x such that x is at least <b>number</b>, and x modulo * <b>divisor</b> == 0. If no such x can be expressed as a uint64_t, return - * UINT64_MAX */ + * UINT64_MAX. Asserts if divisor is zero. */ uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) { @@ -580,6 +572,19 @@ add_laplace_noise(int64_t signal_, double random_, double delta_f, return signal_ + noise; } +/* Helper: safely add two uint32_t's, capping at UINT32_MAX rather + * than overflow */ +uint32_t +tor_add_u32_nowrap(uint32_t a, uint32_t b) +{ + /* a+b > UINT32_MAX check, without overflow */ + if (PREDICT_UNLIKELY(a > UINT32_MAX - b)) { + return UINT32_MAX; + } else { + return a+b; + } +} + /* Helper: return greatest common divisor of a,b */ static uint64_t gcd64(uint64_t a, uint64_t b) @@ -712,6 +717,19 @@ tor_strisnonupper(const char *s) return 1; } +/** Return true iff every character in <b>s</b> is whitespace space; else + * return false. */ +int +tor_strisspace(const char *s) +{ + while (*s) { + if (!TOR_ISSPACE(*s)) + return 0; + s++; + } + return 1; +} + /** As strcmp, except that either string may be NULL. The NULL string is * considered to be before any non-NULL string. */ int @@ -1066,6 +1084,36 @@ string_is_valid_ipv6_address(const char *string) return (tor_inet_pton(AF_INET6,string,&addr) == 1); } +/** Return true iff <b>string</b> is a valid destination address, + * i.e. either a DNS hostname or IPv4/IPv6 address string. + */ +int +string_is_valid_dest(const char *string) +{ + char *tmp = NULL; + int retval; + size_t len; + + if (string == NULL) + return 0; + + len = strlen(string); + + if (len == 0) + return 0; + + if (string[0] == '[' && string[len - 1] == ']') + string = tmp = tor_strndup(string + 1, len - 2); + + retval = string_is_valid_ipv4_address(string) || + string_is_valid_ipv6_address(string) || + string_is_valid_nonrfc_hostname(string); + + tor_free(tmp); + + return retval; +} + /** Return true iff <b>string</b> matches a pattern of DNS names * that we allow Tor clients to connect to. * @@ -1073,37 +1121,51 @@ string_is_valid_ipv6_address(const char *string) * with misconfigured zones that have been encountered in the wild. */ int -string_is_valid_hostname(const char *string) +string_is_valid_nonrfc_hostname(const char *string) { int result = 1; + int has_trailing_dot; + char *last_label; smartlist_t *components; + if (!string || strlen(string) == 0) + return 0; + + if (string_is_valid_ipv4_address(string)) + return 0; + components = smartlist_new(); smartlist_split_string(components,string,".",0,0); + if (BUG(smartlist_len(components) == 0)) + return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks. + + /* Allow a single terminating '.' used rarely to indicate domains + * are FQDNs rather than relative. */ + last_label = (char *)smartlist_get(components, + smartlist_len(components) - 1); + has_trailing_dot = (last_label[0] == '\0'); + if (has_trailing_dot) { + smartlist_pop_last(components); + tor_free(last_label); + last_label = NULL; + } + SMARTLIST_FOREACH_BEGIN(components, char *, c) { if ((c[0] == '-') || (*c == '_')) { result = 0; break; } - /* Allow a single terminating '.' used rarely to indicate domains - * are FQDNs rather than relative. */ - if ((c_sl_idx > 0) && (c_sl_idx + 1 == c_sl_len) && !*c) { - continue; - } - do { - if ((*c >= 'a' && *c <= 'z') || - (*c >= 'A' && *c <= 'Z') || - (*c >= '0' && *c <= '9') || - (*c == '-') || (*c == '_')) - c++; - else - result = 0; + result = (TOR_ISALNUM(*c) || (*c == '-') || (*c == '_')); + c++; } while (result && *c); + if (result == 0) { + break; + } } SMARTLIST_FOREACH_END(c); SMARTLIST_FOREACH_BEGIN(components, char *, c) { @@ -1167,7 +1229,7 @@ tor_parse_long(const char *s, int base, long min, long max, char *endptr; long r; - if (base < 0) { + if (BUG(base < 0)) { if (ok) *ok = 0; return 0; @@ -1186,7 +1248,7 @@ tor_parse_ulong(const char *s, int base, unsigned long min, char *endptr; unsigned long r; - if (base < 0) { + if (BUG(base < 0)) { if (ok) *ok = 0; return 0; @@ -1218,7 +1280,7 @@ tor_parse_uint64(const char *s, int base, uint64_t min, char *endptr; uint64_t r; - if (base < 0) { + if (BUG(base < 0)) { if (ok) *ok = 0; return 0; @@ -1228,20 +1290,12 @@ tor_parse_uint64(const char *s, int base, uint64_t min, #ifdef HAVE_STRTOULL r = (uint64_t)strtoull(s, &endptr, base); #elif defined(_WIN32) -#if defined(_MSC_VER) && _MSC_VER < 1300 - tor_assert(base <= 10); - r = (uint64_t)_atoi64(s); - endptr = (char*)s; - while (TOR_ISSPACE(*endptr)) endptr++; - while (TOR_ISDIGIT(*endptr)) endptr++; -#else r = (uint64_t)_strtoui64(s, &endptr, base); -#endif #elif SIZEOF_LONG == 8 r = (uint64_t)strtoul(s, &endptr, base); #else #error "I don't know how to parse 64-bit numbers." -#endif +#endif /* defined(HAVE_STRTOULL) || ... */ CHECK_STRTOX_RESULT(); } @@ -1639,7 +1693,7 @@ tor_timegm(const struct tm *tm, time_t *time_out) log_warn(LD_BUG, "Result does not fit in tor_timegm"); return -1; } -#endif +#endif /* SIZEOF_TIME_T < 8 */ *time_out = (time_t)seconds; return 0; } @@ -1780,6 +1834,15 @@ format_iso_time(char *buf, time_t t) strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_gmtime_r(&t, &tm)); } +/** As format_local_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid + * embedding an internal space. */ +void +format_local_iso_time_nospace(char *buf, time_t t) +{ + format_local_iso_time(buf, t); + buf[10] = 'T'; +} + /** As format_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid * embedding an internal space. */ void @@ -1803,17 +1866,26 @@ format_iso_time_nospace_usec(char *buf, const struct timeval *tv) /** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>, * parse it and store its value in *<b>t</b>. Return 0 on success, -1 on * failure. Ignore extraneous stuff in <b>cp</b> after the end of the time - * string, unless <b>strict</b> is set. */ + * string, unless <b>strict</b> is set. If <b>nospace</b> is set, + * expect the YYYY-MM-DDTHH:MM:SS format. */ int -parse_iso_time_(const char *cp, time_t *t, int strict) +parse_iso_time_(const char *cp, time_t *t, int strict, int nospace) { struct tm st_tm; unsigned int year=0, month=0, day=0, hour=0, minute=0, second=0; int n_fields; - char extra_char; - n_fields = tor_sscanf(cp, "%u-%2u-%2u %2u:%2u:%2u%c", &year, &month, - &day, &hour, &minute, &second, &extra_char); - if (strict ? (n_fields != 6) : (n_fields < 6)) { + char extra_char, separator_char; + n_fields = tor_sscanf(cp, "%u-%2u-%2u%c%2u:%2u:%2u%c", + &year, &month, &day, + &separator_char, + &hour, &minute, &second, &extra_char); + if (strict ? (n_fields != 7) : (n_fields < 7)) { + char *esc = esc_for_log(cp); + log_warn(LD_GENERAL, "ISO time %s was unparseable", esc); + tor_free(esc); + return -1; + } + if (separator_char != (nospace ? 'T' : ' ')) { char *esc = esc_for_log(cp); log_warn(LD_GENERAL, "ISO time %s was unparseable", esc); tor_free(esc); @@ -1855,7 +1927,16 @@ parse_iso_time_(const char *cp, time_t *t, int strict) int parse_iso_time(const char *cp, time_t *t) { - return parse_iso_time_(cp, t, 1); + return parse_iso_time_(cp, t, 1, 0); +} + +/** + * As parse_iso_time, but parses a time encoded by format_iso_time_nospace(). + */ +int +parse_iso_time_nospace(const char *cp, time_t *t) +{ + return parse_iso_time_(cp, t, 1, 1); } /** Given a <b>date</b> in one of the three formats allowed by HTTP (ugh), @@ -1930,7 +2011,7 @@ parse_http_time(const char *date, struct tm *tm) /** Given an <b>interval</b> in seconds, try to write it to the * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form. - * Return 0 on success, -1 on failure. + * Returns a non-negative integer on success, -1 on failure. */ int format_time_interval(char *out, size_t out_len, long interval) @@ -1999,7 +2080,7 @@ update_approx_time(time_t now) { cached_approx_time = now; } -#endif +#endif /* !defined(TIME_IS_FAST) */ /* ===== * Rate limiting @@ -2095,7 +2176,7 @@ read_all(tor_socket_t fd, char *buf, size_t count, int isSocket) return -1; } - while (numread != count) { + while (numread < count) { if (isSocket) result = tor_socket_recv(fd, buf+numread, count-numread, 0); else @@ -2128,9 +2209,9 @@ clean_name_for_stat(char *name) return; name[len-1]='\0'; } -#else +#else /* !(defined(_WIN32)) */ (void)name; -#endif +#endif /* defined(_WIN32) */ } /** Wrapper for unlink() to make it mockable for the test suite; returns 0 @@ -2270,10 +2351,14 @@ check_private_dir,(const char *dirname, cpd_check_t check, * permissions on the directory will be checked again below.*/ fd = open(sandbox_intern_string(dirname), O_NOFOLLOW); - if (fd == -1) + if (fd == -1) { + log_warn(LD_FS, "Could not reopen recently created directory %s: %s", + dirname, + strerror(errno)); return -1; - else + } else { close(fd); + } } else if (!(check & CPD_CHECK)) { log_warn(LD_FS, "Directory %s does not exist.", dirname); @@ -2323,21 +2408,27 @@ check_private_dir,(const char *dirname, cpd_check_t check, running_gid = getgid(); } if (st.st_uid != running_uid) { - const struct passwd *pw_uid = NULL; - char *process_ownername = NULL; + char *process_ownername = NULL, *file_ownername = NULL; - pw_uid = tor_getpwuid(running_uid); - process_ownername = pw_uid ? tor_strdup(pw_uid->pw_name) : - tor_strdup("<unknown>"); + { + const struct passwd *pw_running = tor_getpwuid(running_uid); + process_ownername = pw_running ? tor_strdup(pw_running->pw_name) : + tor_strdup("<unknown>"); + } - pw_uid = tor_getpwuid(st.st_uid); + { + const struct passwd *pw_stat = tor_getpwuid(st.st_uid); + file_ownername = pw_stat ? tor_strdup(pw_stat->pw_name) : + tor_strdup("<unknown>"); + } log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by " "%s (%d). Perhaps you are running Tor as the wrong user?", - dirname, process_ownername, (int)running_uid, - pw ? pw->pw_name : "<unknown>", (int)st.st_uid); + dirname, process_ownername, (int)running_uid, + file_ownername, (int)st.st_uid); tor_free(process_ownername); + tor_free(file_ownername); close(fd); return -1; } @@ -2394,7 +2485,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, } } close(fd); -#else +#else /* !(!defined(_WIN32)) */ /* Win32 case: we can't open() a directory. */ (void)effective_user; @@ -2428,7 +2519,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, return -1; } -#endif +#endif /* !defined(_WIN32) */ return 0; } @@ -2448,7 +2539,7 @@ write_str_to_file,(const char *fname, const char *str, int bin)) "We're writing a text string that already contains a CR to %s", escaped(fname)); } -#endif +#endif /* defined(_WIN32) */ return write_bytes_to_file(fname, str, strlen(str), bin); } @@ -2601,6 +2692,14 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write) if (file_data->rename_on_close) { tor_assert(file_data->tempname && file_data->filename); + if (!abort_write) { + tor_assert(strcmp(file_data->filename, file_data->tempname)); + if (replace_file(file_data->tempname, file_data->filename)) { + log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename, + strerror(errno)); + abort_write = r = -1; + } + } if (abort_write) { int res = unlink(file_data->tempname); if (res != 0) { @@ -2609,13 +2708,6 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write) file_data->tempname, strerror(errno)); r = -1; } - } else { - tor_assert(strcmp(file_data->filename, file_data->tempname)); - if (replace_file(file_data->tempname, file_data->filename)) { - log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename, - strerror(errno)); - r = -1; - } } } @@ -2847,7 +2939,7 @@ read_file_to_str, (const char *filename, int flags, struct stat *stat_out)) errno = save_errno; return string; } -#endif +#endif /* !defined(_WIN32) */ if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) { close(fd); @@ -2880,7 +2972,7 @@ read_file_to_str, (const char *filename, int flags, struct stat *stat_out)) if (!bin) { statbuf.st_size = (size_t) r; } else -#endif +#endif /* defined(_WIN32) || defined(__CYGWIN__) */ if (r != statbuf.st_size) { /* Unless we're using text mode on win32, we'd better have an exact * match for size. */ @@ -2954,8 +3046,9 @@ unescape_string(const char *s, char **result, size_t *size_out) *out = '\0'; if (size_out) *size_out = out - *result; return cp+1; - case '\0': + /* LCOV_EXCL_START -- we caught this in parse_config_from_line. */ + case '\0': tor_fragile_assert(); tor_free(*result); return NULL; @@ -3003,8 +3096,9 @@ unescape_string(const char *s, char **result, size_t *size_out) *out++ = cp[1]; cp += 2; break; - default: + /* LCOV_EXCL_START */ + default: /* we caught this above in the initial loop. */ tor_assert_nonfatal_unreached(); tor_free(*result); return NULL; @@ -3017,135 +3111,39 @@ unescape_string(const char *s, char **result, size_t *size_out) } } -/** Given a string containing part of a configuration file or similar format, - * advance past comments and whitespace and try to parse a single line. If we - * parse a line successfully, set *<b>key_out</b> to a new string holding the - * key portion and *<b>value_out</b> to a new string holding the value portion - * of the line, and return a pointer to the start of the next line. If we run - * out of data, return a pointer to the end of the string. If we encounter an - * error, return NULL and set *<b>err_out</b> (if provided) to an error - * message. - */ -const char * -parse_config_line_from_str_verbose(const char *line, char **key_out, - char **value_out, - const char **err_out) +/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the + * enclosing quotes. Backslashes are not unescaped. Return the unquoted + * <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */ +char * +get_unquoted_path(const char *path) { - /* - See torrc_format.txt for a description of the (silly) format this parses. - */ - const char *key, *val, *cp; - int continuation = 0; - - tor_assert(key_out); - tor_assert(value_out); + size_t len = strlen(path); - *key_out = *value_out = NULL; - key = val = NULL; - /* Skip until the first keyword. */ - while (1) { - while (TOR_ISSPACE(*line)) - ++line; - if (*line == '#') { - while (*line && *line != '\n') - ++line; - } else { - break; - } + if (len == 0) { + return tor_strdup(""); } - if (!*line) { /* End of string? */ - *key_out = *value_out = NULL; - return line; + int has_start_quote = (path[0] == '\"'); + int has_end_quote = (len > 0 && path[len-1] == '\"'); + if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) { + return NULL; } - /* Skip until the next space or \ followed by newline. */ - key = line; - while (*line && !TOR_ISSPACE(*line) && *line != '#' && - ! (line[0] == '\\' && line[1] == '\n')) - ++line; - *key_out = tor_strndup(key, line-key); - - /* Skip until the value. */ - while (*line == ' ' || *line == '\t') - ++line; - - val = line; - - /* Find the end of the line. */ - if (*line == '\"') { // XXX No continuation handling is done here - if (!(line = unescape_string(line, value_out, NULL))) { - if (err_out) - *err_out = "Invalid escape sequence in quoted string"; - return NULL; - } - while (*line == ' ' || *line == '\t') - ++line; - if (*line == '\r' && *(++line) == '\n') - ++line; - if (*line && *line != '#' && *line != '\n') { - if (err_out) - *err_out = "Excess data after quoted string"; + char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1); + char *s = unquoted_path; + size_t i; + for (i = has_start_quote; i < len - has_end_quote; i++) { + if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) { + *(s-1) = path[i]; + } else if (path[i] != '\"') { + *s++ = path[i]; + } else { /* unescaped quote */ + tor_free(unquoted_path); return NULL; } - } else { - /* Look for the end of the line. */ - while (*line && *line != '\n' && (*line != '#' || continuation)) { - if (*line == '\\' && line[1] == '\n') { - continuation = 1; - line += 2; - } else if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); - if (*line == '\n') - ++line; - } else { - ++line; - } - } - - if (*line == '\n') { - cp = line++; - } else { - cp = line; - } - /* Now back cp up to be the last nonspace character */ - while (cp>val && TOR_ISSPACE(*(cp-1))) - --cp; - - tor_assert(cp >= val); - - /* Now copy out and decode the value. */ - *value_out = tor_strndup(val, cp-val); - if (continuation) { - char *v_out, *v_in; - v_out = v_in = *value_out; - while (*v_in) { - if (*v_in == '#') { - do { - ++v_in; - } while (*v_in && *v_in != '\n'); - if (*v_in == '\n') - ++v_in; - } else if (v_in[0] == '\\' && v_in[1] == '\n') { - v_in += 2; - } else { - *v_out++ = *v_in++; - } - } - *v_out = '\0'; - } - } - - if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); } - while (TOR_ISSPACE(*line)) ++line; - - return line; + *s = '\0'; + return unquoted_path; } /** Expand any homedir prefix on <b>filename</b>; return a newly allocated @@ -3160,7 +3158,7 @@ expand_filename(const char *filename) * Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/ */ return tor_strdup(filename); -#else +#else /* !(defined(_WIN32)) */ if (*filename == '~') { char *home, *result=NULL; const char *rest; @@ -3190,10 +3188,10 @@ expand_filename(const char *filename) } tor_free(username); rest = slash ? (slash+1) : ""; -#else +#else /* !(defined(HAVE_PWD_H)) */ log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h"); return tor_strdup(filename); -#endif +#endif /* defined(HAVE_PWD_H) */ } tor_assert(home); /* Remove trailing slash. */ @@ -3206,7 +3204,7 @@ expand_filename(const char *filename) } else { return tor_strdup(filename); } -#endif +#endif /* defined(_WIN32) */ } #define MAX_SCANF_WIDTH 9999 @@ -3534,6 +3532,17 @@ smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern, smartlist_add(sl, str); } +/** Append a copy of string to sl */ +void +smartlist_add_strdup(struct smartlist_t *sl, const char *string) +{ + char *copy; + + copy = tor_strdup(string); + + smartlist_add(sl, copy); +} + /** Return a new list containing the filenames in the directory <b>dirname</b>. * Return NULL on error or if <b>dirname</b> is not a directory. */ @@ -3564,10 +3573,10 @@ tor_listdir, (const char *dirname)) name[sizeof(name)-1] = '\0'; #else strlcpy(name,findData.cFileName,sizeof(name)); -#endif +#endif /* defined(UNICODE) */ if (strcmp(name, ".") && strcmp(name, "..")) { - smartlist_add(result, tor_strdup(name)); + smartlist_add_strdup(result, name); } if (!FindNextFile(handle, &findData)) { DWORD err; @@ -3581,7 +3590,7 @@ tor_listdir, (const char *dirname)) } FindClose(handle); tor_free(pattern); -#else +#else /* !(defined(_WIN32)) */ const char *prot_dname = sandbox_intern_string(dirname); DIR *d; struct dirent *de; @@ -3593,10 +3602,10 @@ tor_listdir, (const char *dirname)) if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) continue; - smartlist_add(result, tor_strdup(de->d_name)); + smartlist_add_strdup(result, de->d_name); } closedir(d); -#endif +#endif /* defined(_WIN32) */ return result; } @@ -3612,7 +3621,7 @@ path_is_relative(const char *filename) else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) && filename[1] == ':' && filename[2] == '\\') return 0; -#endif +#endif /* defined(_WIN32) */ else return 1; } @@ -3647,14 +3656,14 @@ start_daemon(void) if (pipe(daemon_filedes)) { /* LCOV_EXCL_START */ log_err(LD_GENERAL,"pipe failed; exiting. Error was %s", strerror(errno)); - exit(1); + exit(1); // exit ok: during daemonize, pipe failed. /* LCOV_EXCL_STOP */ } pid = fork(); if (pid < 0) { /* LCOV_EXCL_START */ log_err(LD_GENERAL,"fork failed. Exiting."); - exit(1); + exit(1); // exit ok: during daemonize, fork failed /* LCOV_EXCL_STOP */ } if (pid) { /* Parent */ @@ -3669,13 +3678,13 @@ start_daemon(void) } fflush(stdout); if (ok == 1) - exit(0); + exit(0); // exit ok: during daemonize, daemonizing. else - exit(1); /* child reported error */ + exit(1); /* child reported error. exit ok: daemonize failed. */ } else { /* Child */ close(daemon_filedes[0]); /* we only write */ - pid = setsid(); /* Detach from controlling terminal */ + (void) setsid(); /* Detach from controlling terminal */ /* * Fork one more time, so the parent (the session group leader) can exit. * This means that we, as a non-session group leader, can never regain a @@ -3683,7 +3692,7 @@ start_daemon(void) * _Advanced Programming in the Unix Environment_. */ if (fork() != 0) { - exit(0); + exit(0); // exit ok: during daemonize, fork failed (2) } set_main_thread(); /* We are now the main thread. */ @@ -3712,14 +3721,14 @@ finish_daemon(const char *desired_cwd) /* Don't hold the wrong FS mounted */ if (chdir(desired_cwd) < 0) { log_err(LD_GENERAL,"chdir to \"%s\" failed. Exiting.",desired_cwd); - exit(1); + exit(1); // exit ok: during daemonize, chdir failed. } nullfd = tor_open_cloexec("/dev/null", O_RDWR, 0); if (nullfd < 0) { /* LCOV_EXCL_START */ log_err(LD_GENERAL,"/dev/null can't be opened. Exiting."); - exit(1); + exit(1); // exit ok: during daemonize, couldn't open /dev/null /* LCOV_EXCL_STOP */ } /* close fds linking to invoking terminal, but @@ -3731,7 +3740,7 @@ finish_daemon(const char *desired_cwd) dup2(nullfd,2) < 0) { /* LCOV_EXCL_START */ log_err(LD_GENERAL,"dup2 failed. Exiting."); - exit(1); + exit(1); // exit ok: during daemonize, dup2 failed. /* LCOV_EXCL_STOP */ } if (nullfd > 2) @@ -3742,7 +3751,7 @@ finish_daemon(const char *desired_cwd) } close(daemon_filedes[1]); } -#else +#else /* !(!defined(_WIN32)) */ /* defined(_WIN32) */ void start_daemon(void) @@ -3753,11 +3762,12 @@ finish_daemon(const char *cp) { (void)cp; } -#endif +#endif /* !defined(_WIN32) */ /** Write the current process ID, followed by NL, into <b>filename</b>. + * Return 0 on success, -1 on failure. */ -void +int write_pidfile(const char *filename) { FILE *pidfile; @@ -3765,13 +3775,19 @@ write_pidfile(const char *filename) if ((pidfile = fopen(filename, "w")) == NULL) { log_warn(LD_FS, "Unable to open \"%s\" for writing: %s", filename, strerror(errno)); + return -1; } else { #ifdef _WIN32 - fprintf(pidfile, "%d\n", (int)_getpid()); + int pid = (int)_getpid(); #else - fprintf(pidfile, "%d\n", (int)getpid()); + int pid = (int)getpid(); #endif - fclose(pidfile); + int rv = 0; + if (fprintf(pidfile, "%d\n", pid) < 0) + rv = -1; + if (fclose(pidfile) < 0) + rv = -1; + return rv; } } @@ -3788,7 +3804,7 @@ load_windows_system_library(const TCHAR *library_name) _tcscat(path, library_name); return LoadLibrary(path); } -#endif +#endif /* defined(_WIN32) */ /** Format a single argument for being put on a Windows command line. * Returns a newly allocated string */ @@ -3948,7 +3964,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, * call it with a signed int and an unsigned char, and since the C standard * does not guarantee that an int is wider than a char (an int must be at * least 16 bits but it is permitted for a char to be that wide as well), we - * can't assume a signed int is sufficient to accomodate an unsigned char. + * can't assume a signed int is sufficient to accommodate an unsigned char. * Thus, format_helper_exit_status() will still need to emit any require '-' * on its own. * @@ -3978,7 +3994,7 @@ format_dec_number_sigsafe(unsigned long x, char *buf, int buf_len) * * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded * with spaces. CHILD_STATE indicates where - * in the processs of starting the child process did the failure occur (see + * in the process of starting the child process did the failure occur (see * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of * errno when the failure occurred. * @@ -4083,7 +4099,7 @@ format_helper_exit_status(unsigned char child_state, int saved_errno, done: return res; } -#endif +#endif /* !defined(_WIN32) */ /* Maximum number of file descriptors, if we cannot get it via sysconf() */ #define DEFAULT_MAX_FD 256 @@ -4107,12 +4123,12 @@ tor_terminate_process(process_handle_t *process_handle) else return 0; } -#else /* Unix */ +#else /* !(defined(_WIN32)) */ if (process_handle->waitpid_cb) { /* We haven't got a waitpid yet, so we can just kill off the process. */ return kill(process_handle->pid, SIGTERM); } -#endif +#endif /* defined(_WIN32) */ return 0; /* We didn't need to kill the process, so report success */ } @@ -4134,14 +4150,14 @@ tor_process_get_stdout_pipe(process_handle_t *process_handle) { return process_handle->stdout_pipe; } -#else +#else /* !(defined(_WIN32)) */ /* DOCDOC tor_process_get_stdout_pipe */ -FILE * +int tor_process_get_stdout_pipe(process_handle_t *process_handle) { - return process_handle->stdout_handle; + return process_handle->stdout_pipe; } -#endif +#endif /* defined(_WIN32) */ /* DOCDOC process_handle_new */ static process_handle_t * @@ -4157,7 +4173,7 @@ process_handle_new(void) out->stdin_pipe = -1; out->stdout_pipe = -1; out->stderr_pipe = -1; -#endif +#endif /* defined(_WIN32) */ return out; } @@ -4177,7 +4193,7 @@ process_handle_waitpid_cb(int status, void *arg) process_handle->status = PROCESS_STATUS_NOTRUNNING; process_handle->waitpid_cb = 0; } -#endif +#endif /* !defined(_WIN32) */ /** * @name child-process states @@ -4199,6 +4215,20 @@ process_handle_waitpid_cb(int status, void *arg) #define CHILD_STATE_EXEC 8 #define CHILD_STATE_FAILEXEC 9 /** @} */ +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} /** Start a program in the background. If <b>filename</b> contains a '/', then * it will be treated as an absolute or relative path. Otherwise, on * non-Windows systems, the system path will be searched for <b>filename</b>. @@ -4223,6 +4253,12 @@ tor_spawn_background(const char *const filename, const char **argv, process_environment_t *env, process_handle_t **process_handle_out) { + if (BUG(may_spawn_background_process == 0)) { + /* We should never reach this point if we're forbidden to spawn + * processes. Instead we should have caught the attempt earlier. */ + return PROCESS_STATUS_ERROR; + } + #ifdef _WIN32 HANDLE stdout_pipe_read = NULL; HANDLE stdout_pipe_write = NULL; @@ -4340,13 +4376,12 @@ tor_spawn_background(const char *const filename, const char **argv, /* TODO: Close pipes on exit */ *process_handle_out = process_handle; return status; -#else // _WIN32 +#else /* !(defined(_WIN32)) */ pid_t pid; int stdout_pipe[2]; int stderr_pipe[2]; int stdin_pipe[2]; int fd, retval; - ssize_t nbytes; process_handle_t *process_handle; int status; @@ -4367,7 +4402,7 @@ tor_spawn_background(const char *const filename, const char **argv, and we are not allowed to use unsafe functions between fork and exec */ error_message_length = strlen(error_message); - child_state = CHILD_STATE_PIPE; + // child_state = CHILD_STATE_PIPE; /* Set up pipe for redirecting stdout, stderr, and stdin of child */ retval = pipe(stdout_pipe); @@ -4404,7 +4439,7 @@ tor_spawn_background(const char *const filename, const char **argv, return status; } - child_state = CHILD_STATE_MAXFD; + // child_state = CHILD_STATE_MAXFD; #ifdef _SC_OPEN_MAX if (-1 == max_fd) { @@ -4415,11 +4450,11 @@ tor_spawn_background(const char *const filename, const char **argv, "Cannot find maximum file descriptor, assuming %d", max_fd); } } -#else +#else /* !(defined(_SC_OPEN_MAX)) */ max_fd = DEFAULT_MAX_FD; -#endif +#endif /* defined(_SC_OPEN_MAX) */ - child_state = CHILD_STATE_FORK; + // child_state = CHILD_STATE_FORK; pid = fork(); if (0 == pid) { @@ -4432,7 +4467,7 @@ tor_spawn_background(const char *const filename, const char **argv, * than nothing. */ prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ child_state = CHILD_STATE_DUPOUT; @@ -4455,7 +4490,7 @@ tor_spawn_background(const char *const filename, const char **argv, if (-1 == retval) goto error; - child_state = CHILD_STATE_CLOSEFD; + // child_state = CHILD_STATE_CLOSEFD; close(stderr_pipe[0]); close(stderr_pipe[1]); @@ -4471,7 +4506,7 @@ tor_spawn_background(const char *const filename, const char **argv, close(fd); } - child_state = CHILD_STATE_EXEC; + // child_state = CHILD_STATE_EXEC; /* Call the requested program. We need the cast because execvp doesn't define argv as const, even though it @@ -4490,7 +4525,8 @@ tor_spawn_background(const char *const filename, const char **argv, error: { /* XXX: are we leaking fds from the pipe? */ - int n; + int n, err=0; + ssize_t nbytes; n = format_helper_exit_status(child_state, errno, hex_errno); @@ -4499,13 +4535,14 @@ tor_spawn_background(const char *const filename, const char **argv, value, but there is nothing we can do if it fails */ /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ nbytes = write(STDOUT_FILENO, error_message, error_message_length); + err = (nbytes < 0); nbytes = write(STDOUT_FILENO, hex_errno, n); + err += (nbytes < 0); } - } - (void) nbytes; + _exit(err?254:255); // exit ok: in child. + } - _exit(255); /* Never reached, but avoids compiler warning */ return status; // LCOV_EXCL_LINE } @@ -4570,14 +4607,10 @@ tor_spawn_background(const char *const filename, const char **argv, log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " "nonblocking in parent process: %s", strerror(errno)); } - /* Open the buffered IO streams */ - process_handle->stdout_handle = fdopen(process_handle->stdout_pipe, "r"); - process_handle->stderr_handle = fdopen(process_handle->stderr_pipe, "r"); - process_handle->stdin_handle = fdopen(process_handle->stdin_pipe, "r"); *process_handle_out = process_handle; - return process_handle->status; -#endif // _WIN32 + return status; +#endif /* defined(_WIN32) */ } /** Destroy all resources allocated by the process handle in @@ -4619,18 +4652,13 @@ tor_process_handle_destroy,(process_handle_t *process_handle, if (process_handle->stdin_pipe) CloseHandle(process_handle->stdin_pipe); -#else - if (process_handle->stdout_handle) - fclose(process_handle->stdout_handle); - - if (process_handle->stderr_handle) - fclose(process_handle->stderr_handle); - - if (process_handle->stdin_handle) - fclose(process_handle->stdin_handle); +#else /* !(defined(_WIN32)) */ + close(process_handle->stdout_pipe); + close(process_handle->stderr_pipe); + close(process_handle->stdin_pipe); clear_waitpid_callback(process_handle->waitpid_cb); -#endif +#endif /* defined(_WIN32) */ memset(process_handle, 0x0f, sizeof(process_handle_t)); tor_free(process_handle); @@ -4683,7 +4711,7 @@ tor_get_exit_code(process_handle_t *process_handle, return PROCESS_EXIT_ERROR; } } -#else +#else /* !(defined(_WIN32)) */ int stat_loc; int retval; @@ -4718,7 +4746,7 @@ tor_get_exit_code(process_handle_t *process_handle, if (exit_code != NULL) *exit_code = WEXITSTATUS(stat_loc); -#endif // _WIN32 +#endif /* defined(_WIN32) */ return PROCESS_EXIT_EXITED; } @@ -4751,7 +4779,7 @@ environment_variable_names_equal(const char *s1, const char *s2) /** Free <b>env</b> (assuming it was produced by * process_environment_make). */ void -process_environment_free(process_environment_t *env) +process_environment_free_(process_environment_t *env) { if (env == NULL) return; @@ -4773,8 +4801,8 @@ process_environment_t * process_environment_make(struct smartlist_t *env_vars) { process_environment_t *env = tor_malloc_zero(sizeof(process_environment_t)); - size_t n_env_vars = smartlist_len(env_vars); - size_t i; + int n_env_vars = smartlist_len(env_vars); + int i; size_t total_env_length; smartlist_t *env_vars_sorted; @@ -4864,7 +4892,7 @@ get_current_process_environment_variables(void) char **environ_tmp; /* Not const char ** ? Really? */ for (environ_tmp = get_environment(); *environ_tmp; ++environ_tmp) { - smartlist_add(sl, tor_strdup(*environ_tmp)); + smartlist_add_strdup(sl, *environ_tmp); } return sl; @@ -4913,7 +4941,7 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { + while (numread < count) { /* Check if there is anything to read */ retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); if (!retval) { @@ -4958,20 +4986,20 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, } return (ssize_t)numread; } -#else -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If +#else /* !(defined(_WIN32)) */ +/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If * <b>process</b> is NULL, the function will return immediately if there is * nothing more to read. Otherwise data will be read until end of file, or * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the * file has been reached. */ ssize_t -tor_read_all_handle(FILE *h, char *buf, size_t count, +tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof) { size_t numread = 0; - char *retval; + ssize_t result; if (eof) *eof = 0; @@ -4979,37 +5007,31 @@ tor_read_all_handle(FILE *h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { - /* Use fgets because that is what we use in log_from_pipe() */ - retval = fgets(buf+numread, (int)(count-numread), h); - if (NULL == retval) { - if (feof(h)) { - log_debug(LD_GENERAL, "fgets() reached end of file"); - if (eof) - *eof = 1; + while (numread < count) { + result = read(fd, buf+numread, count-numread); + + if (result == 0) { + log_debug(LD_GENERAL, "read() reached end of file"); + if (eof) + *eof = 1; + break; + } else if (result < 0 && errno == EAGAIN) { + if (process) + continue; + else break; - } else { - if (EAGAIN == errno) { - if (process) - continue; - else - break; - } else { - log_warn(LD_GENERAL, "fgets() from handle failed: %s", - strerror(errno)); - return -1; - } - } + } else if (result < 0) { + log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); + return -1; } - tor_assert(retval != NULL); - tor_assert(strlen(retval) + numread <= count); - numread += strlen(retval); + + numread += result; } - log_debug(LD_GENERAL, "fgets() read %d bytes from handle", (int)numread); + log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); return (ssize_t)numread; } -#endif +#endif /* defined(_WIN32) */ /** Read from stdout of a process until the process exits. */ ssize_t @@ -5020,9 +5042,9 @@ tor_read_all_from_process_stdout(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stdout_handle, buf, count, + return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle, NULL); -#endif +#endif /* defined(_WIN32) */ } /** Read from stdout of a process until the process exits. */ @@ -5034,9 +5056,9 @@ tor_read_all_from_process_stderr(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stderr_handle, buf, count, + return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle, NULL); -#endif +#endif /* defined(_WIN32) */ } /** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be @@ -5111,30 +5133,6 @@ stream_status_to_string(enum stream_status stream_status) } } -/* DOCDOC */ -static void -log_portfw_spawn_error_message(const char *buf, - const char *executable, int *child_status) -{ - /* Parse error message */ - int retval, child_state, saved_errno; - retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Failed to start child process \"%s\" in state %d: %s", - executable, child_state, strerror(saved_errno)); - if (child_status) - *child_status = 1; - } else { - /* Failed to parse message from child process, log it as a - warning */ - log_warn(LD_GENERAL, - "Unexpected message from port forwarding helper \"%s\": %s", - executable, buf); - } -} - #ifdef _WIN32 /** Return a smartlist containing lines outputted from @@ -5180,59 +5178,13 @@ tor_get_lines_from_handle, (HANDLE *handle, return lines; } -/** Read from stream, and send lines to log at the specified log level. - * Returns -1 if there is a error reading, and 0 otherwise. - * If the generated stream is flushed more often than on new lines, or - * a read exceeds 256 bytes, lines will be truncated. This should be fixed, - * along with the corresponding problem on *nix (see bug #2045). - */ -static int -log_from_handle(HANDLE *pipe, int severity) -{ - char buf[256]; - int pos; - smartlist_t *lines; - - pos = tor_read_all_handle(pipe, buf, sizeof(buf) - 1, NULL); - if (pos < 0) { - /* Error */ - log_warn(LD_GENERAL, "Failed to read data from subprocess"); - return -1; - } - - if (0 == pos) { - /* There's nothing to read (process is busy or has exited) */ - log_debug(LD_GENERAL, "Subprocess had nothing to say"); - return 0; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - buf[pos] = '\0'; - log_debug(LD_GENERAL, "Subprocess had %d bytes to say", pos); - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, buf, pos); - - /* Log each line */ - SMARTLIST_FOREACH(lines, char *, line, - { - log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", line); - }); - smartlist_free(lines); - - return 0; -} - -#else +#else /* !(defined(_WIN32)) */ /** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set + * <b>fd</b>. Return NULL on error, and set * <b>stream_status_out</b> appropriately. */ MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (FILE *handle, - enum stream_status *stream_status_out)) +tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) { enum stream_status stream_status; char stdout_buf[400]; @@ -5241,13 +5193,13 @@ tor_get_lines_from_handle, (FILE *handle, while (1) { memset(stdout_buf, 0, sizeof(stdout_buf)); - stream_status = get_string_from_pipe(handle, + stream_status = get_string_from_pipe(fd, stdout_buf, sizeof(stdout_buf) - 1); if (stream_status != IO_STREAM_OKAY) goto done; if (!lines) lines = smartlist_new(); - smartlist_add(lines, tor_strdup(stdout_buf)); + smartlist_split_string(lines, stdout_buf, "\n", 0, 0); } done: @@ -5255,45 +5207,9 @@ tor_get_lines_from_handle, (FILE *handle, return lines; } -/** Read from stream, and send lines to log at the specified log level. - * Returns 1 if stream is closed normally, -1 if there is a error reading, and - * 0 otherwise. Handles lines from tor-fw-helper and - * tor_spawn_background() specially. - */ -static int -log_from_pipe(FILE *stream, int severity, const char *executable, - int *child_status) -{ - char buf[256]; - enum stream_status r; - - for (;;) { - r = get_string_from_pipe(stream, buf, sizeof(buf) - 1); - - if (r == IO_STREAM_CLOSED) { - return 1; - } else if (r == IO_STREAM_EAGAIN) { - return 0; - } else if (r == IO_STREAM_TERM) { - return -1; - } - - tor_assert(r == IO_STREAM_OKAY); - - /* Check if buf starts with SPAWN_ERROR_MESSAGE */ - if (strcmpstart(buf, SPAWN_ERROR_MESSAGE) == 0) { - log_portfw_spawn_error_message(buf, executable, child_status); - } else { - log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf); - } - } - - /* We should never get here */ - return -1; -} -#endif +#endif /* defined(_WIN32) */ -/** Reads from <b>stream</b> and stores input in <b>buf_out</b> making +/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making * sure it's below <b>count</b> bytes. * If the string has a trailing newline, we strip it off. * @@ -5309,340 +5225,28 @@ log_from_pipe(FILE *stream, int severity, const char *executable, * IO_STREAM_OKAY: If everything went okay and we got a string * in <b>buf_out</b>. */ enum stream_status -get_string_from_pipe(FILE *stream, char *buf_out, size_t count) +get_string_from_pipe(int fd, char *buf_out, size_t count) { - char *retval; - size_t len; + ssize_t ret; tor_assert(count <= INT_MAX); - retval = fgets(buf_out, (int)count, stream); + ret = read(fd, buf_out, count); - if (!retval) { - if (feof(stream)) { - /* Program has closed stream (probably it exited) */ - /* TODO: check error */ - return IO_STREAM_CLOSED; - } else { - if (EAGAIN == errno) { - /* Nothing more to read, try again next time */ - return IO_STREAM_EAGAIN; - } else { - /* There was a problem, abandon this child process */ - return IO_STREAM_TERM; - } - } - } else { - len = strlen(buf_out); - if (len == 0) { - /* this probably means we got a NUL at the start of the string. */ - return IO_STREAM_EAGAIN; - } + if (ret == 0) + return IO_STREAM_CLOSED; + else if (ret < 0 && errno == EAGAIN) + return IO_STREAM_EAGAIN; + else if (ret < 0) + return IO_STREAM_TERM; - if (buf_out[len - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[len - 1] = '\0'; - } else { - /* No newline; check whether we overflowed the buffer */ - if (!feof(stream)) - log_info(LD_GENERAL, - "Line from stream was truncated: %s", buf_out); - /* TODO: What to do with this error? */ - } - - return IO_STREAM_OKAY; - } - - /* We should never get here */ - return IO_STREAM_TERM; -} - -/** Parse a <b>line</b> from tor-fw-helper and issue an appropriate - * log message to our user. */ -static void -handle_fw_helper_line(const char *executable, const char *line) -{ - smartlist_t *tokens = smartlist_new(); - char *message = NULL; - char *message_for_log = NULL; - const char *external_port = NULL; - const char *internal_port = NULL; - const char *result = NULL; - int port = 0; - int success = 0; - - if (strcmpstart(line, SPAWN_ERROR_MESSAGE) == 0) { - /* We need to check for SPAWN_ERROR_MESSAGE again here, since it's - * possible that it got sent after we tried to read it in log_from_pipe. - * - * XXX Ideally, we should be using one of stdout/stderr for the real - * output, and one for the output of the startup code. We used to do that - * before cd05f35d2c. - */ - int child_status; - log_portfw_spawn_error_message(line, executable, &child_status); - goto done; - } - - smartlist_split_string(tokens, line, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - - if (smartlist_len(tokens) < 5) - goto err; - - if (strcmp(smartlist_get(tokens, 0), "tor-fw-helper") || - strcmp(smartlist_get(tokens, 1), "tcp-forward")) - goto err; - - external_port = smartlist_get(tokens, 2); - internal_port = smartlist_get(tokens, 3); - result = smartlist_get(tokens, 4); - - if (smartlist_len(tokens) > 5) { - /* If there are more than 5 tokens, they are part of [<message>]. - Let's use a second smartlist to form the whole message; - strncat loops suck. */ - int i; - int message_words_n = smartlist_len(tokens) - 5; - smartlist_t *message_sl = smartlist_new(); - for (i = 0; i < message_words_n; i++) - smartlist_add(message_sl, smartlist_get(tokens, 5+i)); - - tor_assert(smartlist_len(message_sl) > 0); - message = smartlist_join_strings(message_sl, " ", 0, NULL); - - /* wrap the message in log-friendly wrapping */ - tor_asprintf(&message_for_log, " ('%s')", message); - - smartlist_free(message_sl); - } - - port = atoi(external_port); - if (port < 1 || port > 65535) - goto err; - - port = atoi(internal_port); - if (port < 1 || port > 65535) - goto err; - - if (!strcmp(result, "SUCCESS")) - success = 1; - else if (!strcmp(result, "FAIL")) - success = 0; - else - goto err; - - if (!success) { - log_warn(LD_GENERAL, "Tor was unable to forward TCP port '%s' to '%s'%s. " - "Please make sure that your router supports port " - "forwarding protocols (like NAT-PMP). Note that if '%s' is " - "your ORPort, your relay will be unable to receive inbound " - "traffic.", external_port, internal_port, - message_for_log ? message_for_log : "", - internal_port); - } else { - log_info(LD_GENERAL, - "Tor successfully forwarded TCP port '%s' to '%s'%s.", - external_port, internal_port, - message_for_log ? message_for_log : ""); - } - - goto done; - - err: - log_warn(LD_GENERAL, "tor-fw-helper sent us a string we could not " - "parse (%s).", line); - - done: - SMARTLIST_FOREACH(tokens, char *, cp, tor_free(cp)); - smartlist_free(tokens); - tor_free(message); - tor_free(message_for_log); -} - -/** Read what tor-fw-helper has to say in its stdout and handle it - * appropriately */ -static int -handle_fw_helper_output(const char *executable, - process_handle_t *process_handle) -{ - smartlist_t *fw_helper_output = NULL; - enum stream_status stream_status = 0; - - fw_helper_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(process_handle), - &stream_status); - if (!fw_helper_output) { /* didn't get any output from tor-fw-helper */ - /* if EAGAIN we should retry in the future */ - return (stream_status == IO_STREAM_EAGAIN) ? 0 : -1; - } - - /* Handle the lines we got: */ - SMARTLIST_FOREACH_BEGIN(fw_helper_output, char *, line) { - handle_fw_helper_line(executable, line); - tor_free(line); - } SMARTLIST_FOREACH_END(line); - - smartlist_free(fw_helper_output); - - return 0; -} - -/** Spawn tor-fw-helper and ask it to forward the ports in - * <b>ports_to_forward</b>. <b>ports_to_forward</b> contains strings - * of the form "<external port>:<internal port>", which is the format - * that tor-fw-helper expects. */ -void -tor_check_port_forwarding(const char *filename, - smartlist_t *ports_to_forward, - time_t now) -{ -/* When fw-helper succeeds, how long do we wait until running it again */ -#define TIME_TO_EXEC_FWHELPER_SUCCESS 300 -/* When fw-helper failed to start, how long do we wait until running it again - */ -#define TIME_TO_EXEC_FWHELPER_FAIL 60 - - /* Static variables are initialized to zero, so child_handle.status=0 - * which corresponds to it not running on startup */ - static process_handle_t *child_handle=NULL; - - static time_t time_to_run_helper = 0; - int stderr_status, retval; - int stdout_status = 0; - - tor_assert(filename); - - /* Start the child, if it is not already running */ - if ((!child_handle || child_handle->status != PROCESS_STATUS_RUNNING) && - time_to_run_helper < now) { - /*tor-fw-helper cli looks like this: tor_fw_helper -p :5555 -p 4555:1111 */ - const char **argv; /* cli arguments */ - int args_n, status; - int argv_index = 0; /* index inside 'argv' */ - - tor_assert(smartlist_len(ports_to_forward) > 0); - - /* check for overflow during 'argv' allocation: - (len(ports_to_forward)*2 + 2)*sizeof(char*) > SIZE_MAX == - len(ports_to_forward) > (((SIZE_MAX/sizeof(char*)) - 2)/2) */ - if ((size_t) smartlist_len(ports_to_forward) > - (((SIZE_MAX/sizeof(char*)) - 2)/2)) { - log_warn(LD_GENERAL, - "Overflow during argv allocation. This shouldn't happen."); - return; - } - /* check for overflow during 'argv_index' increase: - ((len(ports_to_forward)*2 + 2) > INT_MAX) == - len(ports_to_forward) > (INT_MAX - 2)/2 */ - if (smartlist_len(ports_to_forward) > (INT_MAX - 2)/2) { - log_warn(LD_GENERAL, - "Overflow during argv_index increase. This shouldn't happen."); - return; - } - - /* Calculate number of cli arguments: one for the filename, two - for each smartlist element (one for "-p" and one for the - ports), and one for the final NULL. */ - args_n = 1 + 2*smartlist_len(ports_to_forward) + 1; - argv = tor_calloc(args_n, sizeof(char *)); - - argv[argv_index++] = filename; - SMARTLIST_FOREACH_BEGIN(ports_to_forward, const char *, port) { - argv[argv_index++] = "-p"; - argv[argv_index++] = port; - } SMARTLIST_FOREACH_END(port); - argv[argv_index] = NULL; - - /* Assume tor-fw-helper will succeed, start it later*/ - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_SUCCESS; - - if (child_handle) { - tor_process_handle_destroy(child_handle, 1); - child_handle = NULL; - } - -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - status = tor_spawn_background(NULL, argv, NULL, &child_handle); -#else - status = tor_spawn_background(filename, argv, NULL, &child_handle); -#endif - - tor_free_((void*)argv); - argv=NULL; - - if (PROCESS_STATUS_ERROR == status) { - log_warn(LD_GENERAL, "Failed to start port forwarding helper %s", - filename); - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL; - return; - } - - log_info(LD_GENERAL, - "Started port forwarding helper (%s) with pid '%d'", - filename, tor_process_get_pid(child_handle)); - } - - /* If child is running, read from its stdout and stderr) */ - if (child_handle && PROCESS_STATUS_RUNNING == child_handle->status) { - /* Read from stdout/stderr and log result */ - retval = 0; -#ifdef _WIN32 - stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO); -#else - stderr_status = log_from_pipe(child_handle->stderr_handle, - LOG_INFO, filename, &retval); -#endif - if (handle_fw_helper_output(filename, child_handle) < 0) { - log_warn(LD_GENERAL, "Failed to handle fw helper output."); - stdout_status = -1; - retval = -1; - } - - if (retval) { - /* There was a problem in the child process */ - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL; - } - - /* Combine the two statuses in order of severity */ - if (-1 == stdout_status || -1 == stderr_status) - /* There was a failure */ - retval = -1; -#ifdef _WIN32 - else if (!child_handle || tor_get_exit_code(child_handle, 0, NULL) != - PROCESS_EXIT_RUNNING) { - /* process has exited or there was an error */ - /* TODO: Do something with the process return value */ - /* TODO: What if the process output something since - * between log_from_handle and tor_get_exit_code? */ - retval = 1; - } -#else - else if (1 == stdout_status || 1 == stderr_status) - /* stdout or stderr was closed, the process probably - * exited. It will be reaped by waitpid() in main.c */ - /* TODO: Do something with the process return value */ - retval = 1; -#endif - else - /* Both are fine */ - retval = 0; - - /* If either pipe indicates a failure, act on it */ - if (0 != retval) { - if (1 == retval) { - log_info(LD_GENERAL, "Port forwarding helper terminated"); - child_handle->status = PROCESS_STATUS_NOTRUNNING; - } else { - log_warn(LD_GENERAL, "Failed to read from port forwarding helper"); - child_handle->status = PROCESS_STATUS_ERROR; - } + if (buf_out[ret - 1] == '\n') { + /* Remove the trailing newline */ + buf_out[ret - 1] = '\0'; + } else + buf_out[ret] = '\0'; - /* TODO: The child might not actually be finished (maybe it failed or - closed stdout/stderr), so maybe we shouldn't start another? */ - } - } + return IO_STREAM_OKAY; } /** Initialize the insecure RNG <b>rng</b> from a seed value <b>seed</b>. */ @@ -5695,7 +5299,7 @@ clamp_double_to_int64(double number) { int exponent; -#if (defined(__MINGW32__) || defined(__MINGW64__)) && GCC_VERSION >= 409 +#if defined(MINGW_ANY) && GCC_VERSION >= 409 /* Mingw's math.h uses gcc's __builtin_choose_expr() facility to declare isnan, isfinite, and signbit. But as implemented in at least some @@ -5704,7 +5308,7 @@ clamp_double_to_int64(double number) */ #define PROBLEMATIC_FLOAT_CONVERSION_WARNING DISABLE_GCC_WARNING(float-conversion) -#endif +#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */ /* With clang 4.0 we apparently run into "double promotion" warnings here, @@ -5715,7 +5319,7 @@ DISABLE_GCC_WARNING(float-conversion) #define PROBLEMATIC_DOUBLE_PROMOTION_WARNING DISABLE_GCC_WARNING(double-promotion) #endif -#endif +#endif /* defined(__clang__) */ /* NaN is a special case that can't be used with the logic below. */ if (isnan(number)) { @@ -5762,7 +5366,7 @@ tor_htonll(uint64_t a) /* Little endian. The worst... */ return htonl((uint32_t)(a>>32)) | (((uint64_t)htonl((uint32_t)a))<<32); -#endif /* WORDS_BIGENDIAN */ +#endif /* defined(WORDS_BIGENDIAN) */ } /** Return a uint64_t value from <b>a</b> in host byte order. */ diff --git a/src/common/util.h b/src/common/util.h index 479fc8d610..7172b7da08 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -45,7 +45,7 @@ #else #define DMALLOC_PARAMS #define DMALLOC_ARGS -#endif +#endif /* defined(USE_DMALLOC) */ /* Memory management */ void *tor_malloc_(size_t size DMALLOC_PARAMS) ATTR_MALLOC; @@ -72,21 +72,32 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, (p)=NULL; \ } \ STMT_END -#else -/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, etc. - * Unlike the free() function, tor_free() will still work on NULL pointers, - * and it sets the pointer value to NULL after freeing it. +#else /* !(defined(USE_DMALLOC)) */ +/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, + * etc. Unlike the free() function, the tor_free() macro sets the + * pointer value to NULL after freeing it. * * This is a macro. If you need a function pointer to release memory from * tor_malloc(), use tor_free_(). + * + * Note that this macro takes the address of the pointer it is going to + * free and clear. If that pointer is stored with a nonstandard + * alignment (eg because of a "packed" pragma) it is not correct to use + * tor_free(). */ +#ifdef __GNUC__ +#define tor_free(p) STMT_BEGIN \ + typeof(&(p)) tor_free__tmpvar = &(p); \ + raw_free(*tor_free__tmpvar); \ + *tor_free__tmpvar=NULL; \ + STMT_END +#else #define tor_free(p) STMT_BEGIN \ - if (PREDICT_LIKELY((p)!=NULL)) { \ - raw_free(p); \ - (p)=NULL; \ - } \ + raw_free(p); \ + (p)=NULL; \ STMT_END #endif +#endif /* defined(USE_DMALLOC) */ #define tor_malloc(size) tor_malloc_(size DMALLOC_ARGS) #define tor_malloc_zero(size) tor_malloc_zero_(size DMALLOC_ARGS) @@ -109,19 +120,22 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, void tor_log_mallinfo(int severity); -/** Return the offset of <b>member</b> within the type <b>tp</b>, in bytes */ -#if defined(__GNUC__) && __GNUC__ > 3 -#define STRUCT_OFFSET(tp, member) __builtin_offsetof(tp, member) -#else - #define STRUCT_OFFSET(tp, member) \ - ((off_t) (((char*)&((tp*)0)->member)-(char*)0)) -#endif +/* Helper macro: free a variable of type 'typename' using freefn, and + * set the variable to NULL. + */ +#define FREE_AND_NULL(typename, freefn, var) \ + do { \ + /* only evaluate (var) once. */ \ + typename **tmp__free__ptr ## freefn = &(var); \ + freefn(*tmp__free__ptr ## freefn); \ + (*tmp__free__ptr ## freefn) = NULL; \ + } while (0) /** Macro: yield a pointer to the field at position <b>off</b> within the * structure <b>st</b>. Example: * <pre> * struct a { int foo; int bar; } x; - * off_t bar_offset = STRUCT_OFFSET(struct a, bar); + * off_t bar_offset = offsetof(struct a, bar); * int *bar_p = STRUCT_VAR_P(&x, bar_offset); * *bar_p = 3; * </pre> @@ -138,7 +152,7 @@ void tor_log_mallinfo(int severity); * </pre> */ #define SUBTYPE_P(p, subtype, basemember) \ - ((void*) ( ((char*)(p)) - STRUCT_OFFSET(subtype, basemember) )) + ((void*) ( ((char*)(p)) - offsetof(subtype, basemember) )) /* Logic */ /** Macro: true if two values have the same boolean value. */ @@ -162,10 +176,12 @@ int n_bits_set_u8(uint8_t v); int64_t clamp_double_to_int64(double number); void simplify_fraction64(uint64_t *numer, uint64_t *denom); +uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b); + /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> - * and positive <b>b</b>. Works on integer types only. Not defined if a+b can - * overflow. */ -#define CEIL_DIV(a,b) (((a)+(b)-1)/(b)) + * and positive <b>b</b>. Works on integer types only. Not defined if a+(b-1) + * can overflow. */ +#define CEIL_DIV(a,b) (((a)+((b)-1))/(b)) /* Return <b>v</b> if it's between <b>min</b> and <b>max</b>. Otherwise * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if @@ -186,6 +202,7 @@ void tor_strlower(char *s) ATTR_NONNULL((1)); void tor_strupper(char *s) ATTR_NONNULL((1)); int tor_strisprint(const char *s) ATTR_NONNULL((1)); int tor_strisnonupper(const char *s) ATTR_NONNULL((1)); +int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2)); int strcmp_len(const char *s1, const char *s2, size_t len) ATTR_NONNULL((1,2)); @@ -214,7 +231,8 @@ const char *find_str_at_start_of_line(const char *haystack, const char *needle); int string_is_C_identifier(const char *string); int string_is_key_value(int severity, const char *string); -int string_is_valid_hostname(const char *string); +int string_is_valid_dest(const char *string); +int string_is_valid_nonrfc_hostname(const char *string); int string_is_valid_ipv4_address(const char *string); int string_is_valid_ipv6_address(const char *string); @@ -239,6 +257,7 @@ void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...) void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern, va_list args) CHECK_PRINTF(2, 0); +void smartlist_add_strdup(struct smartlist_t *sl, const char *string); /* Time helpers */ long tv_udiff(const struct timeval *start, const struct timeval *end); @@ -252,10 +271,12 @@ int parse_rfc1123_time(const char *buf, time_t *t); #define ISO_TIME_USEC_LEN (ISO_TIME_LEN+7) void format_local_iso_time(char *buf, time_t t); void format_iso_time(char *buf, time_t t); +void format_local_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace_usec(char *buf, const struct timeval *tv); -int parse_iso_time_(const char *cp, time_t *t, int strict); +int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace); int parse_iso_time(const char *buf, time_t *t); +int parse_iso_time_nospace(const char *cp, time_t *t); int parse_http_time(const char *buf, struct tm *tm); int format_time_interval(char *out, size_t out_len, long interval); @@ -266,7 +287,7 @@ int format_time_interval(char *out, size_t out_len, long interval); #else time_t approx_time(void); void update_approx_time(time_t now); -#endif +#endif /* defined(TIME_IS_FAST) */ /* Rate-limiter */ @@ -319,7 +340,7 @@ enum stream_status { const char *stream_status_to_string(enum stream_status stream_status); -enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count); +enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); MOCK_DECL(int,tor_unlink,(const char *pathname)); @@ -386,9 +407,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out) ATTR_MALLOC; const char *unescape_string(const char *s, char **result, size_t *size_out); -const char *parse_config_line_from_str_verbose(const char *line, - char **key_out, char **value_out, - const char **err_out); +char *get_unquoted_path(const char *path); char *expand_filename(const char *filename); MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); int path_is_relative(const char *filename); @@ -396,12 +415,9 @@ int path_is_relative(const char *filename); /* Process helpers */ void start_daemon(void); void finish_daemon(const char *desired_cwd); -void write_pidfile(const char *filename); +int write_pidfile(const char *filename); -/* Port forwarding */ -void tor_check_port_forwarding(const char *filename, - struct smartlist_t *ports_to_forward, - time_t now); +void tor_disable_spawning_background_processes(void); typedef struct process_handle_t process_handle_t; typedef struct process_environment_t process_environment_t; @@ -428,7 +444,9 @@ struct process_environment_t { }; process_environment_t *process_environment_make(struct smartlist_t *env_vars); -void process_environment_free(process_environment_t *env); +void process_environment_free_(process_environment_t *env); +#define process_environment_free(env) \ + FREE_AND_NULL(process_environment_t, process_environment_free_, (env)) struct smartlist_t *get_current_process_environment_variables(void); @@ -437,9 +455,7 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars, void (*free_old)(void*), int free_p); -/* Values of process_handle_t.status. PROCESS_STATUS_NOTRUNNING must be - * 0 because tor_check_port_forwarding depends on this being the initial - * statue of the static instance of process_handle_t */ +/* Values of process_handle_t.status. */ #define PROCESS_STATUS_NOTRUNNING 0 #define PROCESS_STATUS_RUNNING 1 #define PROCESS_STATUS_ERROR -1 @@ -456,13 +472,10 @@ struct process_handle_t { HANDLE stdout_pipe; HANDLE stderr_pipe; PROCESS_INFORMATION pid; -#else +#else /* !(defined(_WIN32)) */ int stdin_pipe; int stdout_pipe; int stderr_pipe; - FILE *stdin_handle; - FILE *stdout_handle; - FILE *stderr_handle; pid_t pid; /** If the process has not given us a SIGCHLD yet, this has the * waitpid_callback_t that gets invoked once it has. Otherwise this @@ -470,9 +483,9 @@ struct process_handle_t { struct waitpid_callback_t *waitpid_cb; /** The exit status reported by waitpid. */ int waitpid_exit_status; -#endif // _WIN32 +#endif /* defined(_WIN32) */ }; -#endif +#endif /* defined(UTIL_PRIVATE) */ /* Return values of tor_get_exit_code() */ #define PROCESS_EXIT_RUNNING 1 @@ -485,10 +498,10 @@ int tor_split_lines(struct smartlist_t *sl, char *buf, int len); ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, const process_handle_t *process); #else -ssize_t tor_read_all_handle(FILE *h, char *buf, size_t count, +ssize_t tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof); -#endif +#endif /* defined(_WIN32) */ ssize_t tor_read_all_from_process_stdout( const process_handle_t *process_handle, char *buf, size_t count); ssize_t tor_read_all_from_process_stderr( @@ -499,7 +512,7 @@ int tor_process_get_pid(process_handle_t *process_handle); #ifdef _WIN32 HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); #else -FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle); +int tor_process_get_stdout_pipe(process_handle_t *process_handle); #endif #ifdef _WIN32 @@ -508,9 +521,9 @@ tor_get_lines_from_handle,(HANDLE *handle, enum stream_status *stream_status)); #else MOCK_DECL(struct smartlist_t *, -tor_get_lines_from_handle,(FILE *handle, +tor_get_lines_from_handle,(int fd, enum stream_status *stream_status)); -#endif +#endif /* defined(_WIN32) */ int tor_terminate_process(process_handle_t *process_handle); @@ -547,15 +560,13 @@ STATIC int format_helper_exit_status(unsigned char child_state, leading minus) and newline (no null) */ #define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ 1 + sizeof(int) * 2 + 1) -#endif +#endif /* !defined(_WIN32) */ -#endif +#endif /* defined(UTIL_PRIVATE) */ -#ifdef TOR_UNIT_TESTS -int size_mul_check__(const size_t x, const size_t y); -#endif +int size_mul_check(const size_t x, const size_t y); #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) -#endif +#endif /* !defined(TOR_UTIL_H) */ diff --git a/src/common/util_bug.c b/src/common/util_bug.c index 08aba47974..126e843866 100644 --- a/src/common/util_bug.c +++ b/src/common/util_bug.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,10 @@ #include "backtrace.h" #include "container.h" +#ifdef __COVERITY__ +int bug_macro_deadcode_dummy__ = 0; +#endif + #ifdef TOR_UNIT_TESTS static void (*failed_assertion_cb)(void) = NULL; static int n_bugs_to_capture = 0; @@ -44,7 +48,7 @@ static void add_captured_bug(const char *s) { --n_bugs_to_capture; - smartlist_add(bug_messages, tor_strdup(s)); + smartlist_add_strdup(bug_messages, s); } /** Set a callback to be invoked when we get any tor_bug_occurred_ * invocation. We use this in the unit tests so that a nonfatal @@ -55,10 +59,10 @@ tor_set_failed_assertion_callback(void (*fn)(void)) { failed_assertion_cb = fn; } -#else +#else /* !(defined(TOR_UNIT_TESTS)) */ #define capturing_bugs() (0) #define add_captured_bug(s) do { } while (0) -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Helper for tor_assert: report the assertion failure. */ void diff --git a/src/common/util_bug.h b/src/common/util_bug.h index 0db6bb6ab0..50becd0c33 100644 --- a/src/common/util_bug.h +++ b/src/common/util_bug.h @@ -1,10 +1,36 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file util_bug.h + * + * \brief Macros to manage assertions, fatal and non-fatal. + * + * Guidelines: All the different kinds of assertion in this file are for + * bug-checking only. Don't write code that can assert based on bad inputs. + * + * We provide two kinds of assertion here: "fatal" and "nonfatal". Use + * nonfatal assertions for any bug you can reasonably recover from -- and + * please, try to recover! Many severe bugs in Tor have been caused by using + * a regular assertion when a nonfatal assertion would have been better. + * + * If you need to check a condition with a nonfatal assertion, AND recover + * from that same condition, consider using the BUG() macro inside a + * conditional. For example: + * + * <code> + * // wrong -- use tor_assert_nonfatal() if you just want an assertion. + * BUG(ptr == NULL); + * + * // okay, but needlessly verbose + * tor_assert_nonfatal(ptr != NULL); + * if (ptr == NULL) { ... } + * + * // this is how we do it: + * if (BUG(ptr == NULL)) { ... } + * </code> **/ #ifndef TOR_UTIL_BUG_H @@ -27,7 +53,7 @@ * security-critical properties. */ #error "Sorry; we don't support building with NDEBUG." -#endif +#endif /* defined(NDEBUG) */ #if defined(TOR_UNIT_TESTS) && defined(__GNUC__) /* We define this GCC macro as a replacement for PREDICT_UNLIKELY() in this @@ -74,9 +100,14 @@ tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ abort(); \ } STMT_END -#endif +#endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */ -#define tor_assert_unreached() tor_assert(0) +#define tor_assert_unreached() \ + STMT_BEGIN { \ + tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, \ + "line should be unreached"); \ + abort(); \ + } STMT_END /* Non-fatal bug assertions. The "unreached" variants mean "this line should * never be reached." The "once" variants mean "Don't log a warning more than @@ -89,11 +120,14 @@ */ #ifdef __COVERITY__ +extern int bug_macro_deadcode_dummy__; #undef BUG // Coverity defines this in global headers; let's override it. This is a // magic coverity-only preprocessor thing. -#nodef BUG(x) ((x)?(__coverity_panic__(),1):0) -#endif +// We use this "deadcode_dummy__" trick to prevent coverity from +// complaining about unreachable bug cases. +#nodef BUG(x) ((x)?(__coverity_panic__(),1):(0+bug_macro_deadcode_dummy__)) +#endif /* defined(__COVERITY__) */ #if defined(__COVERITY__) || defined(__clang_analyzer__) // We're running with a static analysis tool: let's treat even nonfatal @@ -146,7 +180,7 @@ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \ : 0) -#endif +#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */ #ifdef __GNUC__ #define IF_BUG_ONCE__(cond,var) \ @@ -159,7 +193,7 @@ "!("#cond")", 1); \ } \ bool_result; } )) -#else +#else /* !(defined(__GNUC__)) */ #define IF_BUG_ONCE__(cond,var) \ static int var = 0; \ if ((cond) ? \ @@ -169,7 +203,7 @@ "!("#cond")", 1), \ 1)) \ : 0) -#endif +#endif /* defined(__GNUC__) */ #define IF_BUG_ONCE_VARNAME_(a) \ warning_logged_on_ ## a ## __ #define IF_BUG_ONCE_VARNAME__(a) \ @@ -199,6 +233,6 @@ void tor_capture_bugs_(int n); void tor_end_capture_bugs_(void); const struct smartlist_t *tor_get_captured_bug_log_(void); void tor_set_failed_assertion_callback(void (*fn)(void)); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ -#endif +#endif /* !defined(TOR_UTIL_BUG_H) */ diff --git a/src/common/util_format.c b/src/common/util_format.c index aef9db85c8..e51757a4e8 100644 --- a/src/common/util_format.c +++ b/src/common/util_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,13 +22,16 @@ #include <stdlib.h> /* Return the base32 encoded size in bytes using the source length srclen. - * The NUL terminated byte is added as well since every base32 encoding - * requires enough space for it. */ + * + * (WATCH OUT: This API counts the terminating NUL byte, but + * base64_encode_size does not.) + */ size_t base32_encoded_size(size_t srclen) { size_t enclen; - enclen = CEIL_DIV(srclen*8, 5) + 1; + tor_assert(srclen < SIZE_T_CEILING / 8); + enclen = BASE32_NOPAD_BUFSIZE(srclen); tor_assert(enclen < INT_MAX && enclen > srclen); return enclen; } @@ -41,7 +44,6 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) size_t nbits = srclen * 8; size_t bit; - tor_assert(srclen < SIZE_T_CEILING/8); /* We need enough space for the encoded data and the extra NUL byte. */ tor_assert(base32_encoded_size(srclen) <= destlen); tor_assert(destlen < SIZE_T_CEILING); @@ -51,9 +53,10 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) for (i=0,bit=0; bit < nbits; ++i, bit+=5) { /* set v to the 16-bit value starting at src[bits/8], 0-padded. */ - v = ((uint8_t)src[bit/8]) << 8; - if (bit+5<nbits) - v += (uint8_t)src[(bit/8)+1]; + size_t idx = bit / 8; + v = ((uint8_t)src[idx]) << 8; + if (idx+1 < srclen) + v += (uint8_t)src[idx+1]; /* set u to the 5-bit value at the bit'th bit of buf. */ u = (v >> (11-(bit%8))) & 0x1F; dest[i] = BASE32_CHARS[u]; @@ -133,6 +136,9 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) /** Return the Base64 encoded size of <b>srclen</b> bytes of data in * bytes. * + * (WATCH OUT: This API <em>does not</em> count the terminating NUL byte, + * but base32_encoded_size does.) + * * If <b>flags</b>&BASE64_ENCODE_MULTILINE is true, return the size * of the encoded output as multiline output (64 character, `\n' terminated * lines). @@ -141,19 +147,16 @@ size_t base64_encode_size(size_t srclen, int flags) { size_t enclen; + + /* Use INT_MAX for overflow checking because base64_encode() returns int. */ tor_assert(srclen < INT_MAX); + tor_assert(CEIL_DIV(srclen, 3) < INT_MAX / 4); - if (srclen == 0) - return 0; + enclen = BASE64_LEN(srclen); + if (flags & BASE64_ENCODE_MULTILINE) + enclen += CEIL_DIV(enclen, BASE64_OPENSSL_LINELEN); - enclen = ((srclen - 1) / 3) * 4 + 4; - if (flags & BASE64_ENCODE_MULTILINE) { - size_t remainder = enclen % BASE64_OPENSSL_LINELEN; - enclen += enclen / BASE64_OPENSSL_LINELEN; - if (remainder) - enclen++; - } - tor_assert(enclen < INT_MAX && enclen > srclen); + tor_assert(enclen < INT_MAX && (enclen == 0 || enclen > srclen)); return enclen; } @@ -263,10 +266,13 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, ENCODE_N(3); ENCODE_PAD(); break; + // LCOV_EXCL_START -- we can't reach this point, because we enforce + // 0 <= ncov_idx < 3 in the loop above. default: /* Something went catastrophically wrong. */ - tor_fragile_assert(); // LCOV_EXCL_LINE + tor_fragile_assert(); return -1; + // LCOV_EXCL_STOP } #undef ENCODE_N @@ -310,39 +316,6 @@ base64_encode_nopad(char *dest, size_t destlen, return (int)(out - dest); } -/** As base64_decode, but do not require any padding on the input */ -int -base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen) -{ - if (srclen > SIZE_T_CEILING - 4) - return -1; - char *buf = tor_malloc(srclen + 4); - memcpy(buf, src, srclen+1); - size_t buflen; - switch (srclen % 4) - { - case 0: - default: - buflen = srclen; - break; - case 1: - tor_free(buf); - return -1; - case 2: - memcpy(buf+srclen, "==", 3); - buflen = srclen + 2; - break; - case 3: - memcpy(buf+srclen, "=", 2); - buflen = srclen + 1; - break; - } - int n = base64_decode((char*)dest, destlen, buf, buflen); - tor_free(buf); - return n; -} - #undef BASE64_OPENSSL_LINELEN /** @{ */ @@ -392,15 +365,9 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) const char *eos = src+srclen; uint32_t n=0; int n_idx=0; - char *dest_orig = dest; + size_t di = 0; - /* Max number of bits == srclen*6. - * Number of bytes required to hold all bits == (srclen*6)/8. - * Yes, we want to round down: anything that hangs over the end of a - * byte is padding. */ - if (destlen < (srclen*3)/4) - return -1; - if (destlen > SIZE_T_CEILING) + if (destlen > INT_MAX) return -1; /* Make sure we leave no uninitialized data in the destination buffer. */ @@ -428,9 +395,11 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) n = (n<<6) | v; if ((++n_idx) == 4) { /* We've accumulated 24 bits in n. Flush them. */ - *dest++ = (n>>16); - *dest++ = (n>>8) & 0xff; - *dest++ = (n) & 0xff; + if (destlen < 3 || di > destlen - 3) + return -1; + dest[di++] = (n>>16); + dest[di++] = (n>>8) & 0xff; + dest[di++] = (n) & 0xff; n_idx = 0; n = 0; } @@ -448,18 +417,21 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) return -1; case 2: /* 12 leftover bits: The last 4 are padding and the first 8 are data. */ - *dest++ = n >> 4; + if (destlen < 1 || di > destlen - 1) + return -1; + dest[di++] = n >> 4; break; case 3: /* 18 leftover bits: The last 2 are padding and the first 16 are data. */ - *dest++ = n >> 10; - *dest++ = n >> 2; + if (destlen < 2 || di > destlen - 2) + return -1; + dest[di++] = n >> 10; + dest[di++] = n >> 2; } - tor_assert((dest-dest_orig) <= (ssize_t)destlen); - tor_assert((dest-dest_orig) <= INT_MAX); + tor_assert(di <= destlen); - return (int)(dest-dest_orig); + return (int)di; } #undef X #undef SP @@ -475,7 +447,8 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen) const char *end; char *cp; - tor_assert(destlen >= srclen*2+1); + tor_assert(srclen < SIZE_T_CEILING / 2 - 1); + tor_assert(destlen >= BASE16_BUFSIZE(srclen)); tor_assert(destlen < SIZE_T_CEILING); /* Make sure we leave no uninitialized data in the destination buffer. */ diff --git a/src/common/util_format.h b/src/common/util_format.h index 20ac711d10..0aefe3a44e 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_UTIL_FORMAT_H @@ -10,6 +10,26 @@ #include "testsupport.h" #include "torint.h" +/** @{ */ +/** These macros don't check for overflow. Use them only for constant inputs + * (like array declarations). The *_LEN macros are the raw encoding lengths + * (without terminating NUL), while the *_BUFSIZE macros count the terminating + * NUL. */ +#define BASE64_LEN(n) (CEIL_DIV((n), 3) * 4) +#define BASE32_LEN(n) (CEIL_DIV((n), 5) * 8) +#define BASE16_LEN(n) ((n) * 2) + +#define BASE64_BUFSIZE(n) (BASE64_LEN(n) + 1) +#define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1) +#define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1) + +#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3)) +#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5)) + +#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1) +#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1) +/** @} */ + #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, @@ -17,8 +37,6 @@ int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); int base64_encode_nopad(char *dest, size_t destlen, const uint8_t *src, size_t srclen); -int base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen); /** Characters that can appear (case-insensitively) in a base32 encoding. */ #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567" @@ -30,5 +48,5 @@ int hex_decode_digit(char c); void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); -#endif +#endif /* !defined(TOR_UTIL_FORMAT_H) */ diff --git a/src/common/util_process.c b/src/common/util_process.c index abda63720c..c2826152e9 100644 --- a/src/common/util_process.c +++ b/src/common/util_process.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -154,5 +154,5 @@ notify_pending_waitpid_callbacks(void) } } -#endif +#endif /* !defined(_WIN32) */ diff --git a/src/common/util_process.h b/src/common/util_process.h index d38301a354..c9aa771b77 100644 --- a/src/common/util_process.h +++ b/src/common/util_process.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,7 +20,7 @@ waitpid_callback_t *set_waitpid_callback(pid_t pid, void (*fn)(int, void *), void *arg); void clear_waitpid_callback(waitpid_callback_t *ent); void notify_pending_waitpid_callbacks(void); -#endif +#endif /* !defined(_WIN32) */ -#endif +#endif /* !defined(TOR_UTIL_PROCESS_H) */ diff --git a/src/common/workqueue.c b/src/common/workqueue.c index e1fb663a2a..563a98af96 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -1,3 +1,4 @@ + /* copyright (c) 2013-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -24,12 +25,23 @@ #include "orconfig.h" #include "compat.h" +#include "compat_libevent.h" #include "compat_threads.h" +#include "crypto_rand.h" #include "util.h" #include "workqueue.h" #include "tor_queue.h" #include "torlog.h" +#include <event2/event.h> + +#define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH +#define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW +#define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1) + +TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s); +typedef struct work_tailq_t work_tailq_t; + struct threadpool_s { /** An array of pointers to workerthread_t: one for each running worker * thread. */ @@ -38,8 +50,12 @@ struct threadpool_s { /** Condition variable that we wait on when we have no work, and which * gets signaled when our queue becomes nonempty. */ tor_cond_t condition; - /** Queue of pending work that we have to do. */ - TOR_TAILQ_HEAD(, workqueue_entry_s) work; + /** Queues of pending work that we have to do. The queue with priority + * <b>p</b> is work[p]. */ + work_tailq_t work[WORKQUEUE_N_PRIORITIES]; + + /** Weak RNG, used to decide when to ignore priority. */ + tor_weak_rng_t weak_rng; /** The current 'update generation' of the threadpool. Any thread that is * at an earlier generation needs to run the update function. */ @@ -51,6 +67,9 @@ struct threadpool_s { void (*free_update_arg_fn)(void *); /** Array of n_threads update arguments. */ void **update_args; + /** Event to notice when another thread has sent a reply. */ + struct event *reply_event; + void (*reply_cb)(threadpool_t *); /** Number of elements in threads. */ int n_threads; @@ -66,6 +85,11 @@ struct threadpool_s { void *new_thread_state_arg; }; +/** Used to put a workqueue_priority_t value into a bitfield. */ +#define workqueue_priority_bitfield_t ENUM_BF(workqueue_priority_t) +/** Number of bits needed to hold all legal values of workqueue_priority_t */ +#define WORKQUEUE_PRIORITY_BITS 2 + struct workqueue_entry_s { /** The next workqueue_entry_t that's pending on the same thread or * reply queue. */ @@ -76,6 +100,8 @@ struct workqueue_entry_s { struct threadpool_s *on_pool; /** True iff this entry is waiting for a worker to start processing it. */ uint8_t pending; + /** Priority of this entry. */ + workqueue_priority_bitfield_t priority : WORKQUEUE_PRIORITY_BITS; /** Function to run in the worker thread. */ workqueue_reply_t (*fn)(void *state, void *arg); /** Function to run while processing the reply queue. */ @@ -94,9 +120,7 @@ struct replyqueue_s { alert_sockets_t alert; }; -/** A worker thread represents a single thread in a thread pool. To avoid - * contention, each gets its own queue. This breaks the guarantee that that - * queued work will get executed strictly in order. */ +/** A worker thread represents a single thread in a thread pool. */ typedef struct workerthread_s { /** Which thread it this? In range 0..in_pool->n_threads-1 */ int index; @@ -109,6 +133,8 @@ typedef struct workerthread_s { replyqueue_t *reply_queue; /** The current update generation of this thread */ unsigned generation; + /** One over the probability of taking work from a lower-priority queue. */ + int32_t lower_priority_chance; } workerthread_t; static void queue_reply(replyqueue_t *queue, workqueue_entry_t *work); @@ -125,15 +151,19 @@ workqueue_entry_new(workqueue_reply_t (*fn)(void*, void*), ent->fn = fn; ent->reply_fn = reply_fn; ent->arg = arg; + ent->priority = WQ_PRI_HIGH; return ent; } +#define workqueue_entry_free(ent) \ + FREE_AND_NULL(workqueue_entry_t, workqueue_entry_free_, (ent)) + /** * Release all storage held in <b>ent</b>. Call only when <b>ent</b> is not on * any queue. */ static void -workqueue_entry_free(workqueue_entry_t *ent) +workqueue_entry_free_(workqueue_entry_t *ent) { if (!ent) return; @@ -161,8 +191,9 @@ workqueue_entry_cancel(workqueue_entry_t *ent) int cancelled = 0; void *result = NULL; tor_mutex_acquire(&ent->on_pool->lock); + workqueue_priority_t prio = ent->priority; if (ent->pending) { - TOR_TAILQ_REMOVE(&ent->on_pool->work, ent, next_work); + TOR_TAILQ_REMOVE(&ent->on_pool->work[prio], ent, next_work); cancelled = 1; result = ent->arg; } @@ -180,8 +211,46 @@ workqueue_entry_cancel(workqueue_entry_t *ent) static int worker_thread_has_work(workerthread_t *thread) { - return !TOR_TAILQ_EMPTY(&thread->in_pool->work) || - thread->generation != thread->in_pool->generation; + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + if (!TOR_TAILQ_EMPTY(&thread->in_pool->work[i])) + return 1; + } + return thread->generation != thread->in_pool->generation; +} + +/** Extract the next workqueue_entry_t from the the thread's pool, removing + * it from the relevant queues and marking it as non-pending. + * + * The caller must hold the lock. */ +static workqueue_entry_t * +worker_thread_extract_next_work(workerthread_t *thread) +{ + threadpool_t *pool = thread->in_pool; + work_tailq_t *queue = NULL, *this_queue; + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + this_queue = &pool->work[i]; + if (!TOR_TAILQ_EMPTY(this_queue)) { + queue = this_queue; + if (! tor_weak_random_one_in_n(&pool->weak_rng, + thread->lower_priority_chance)) { + /* Usually we'll just break now, so that we can get out of the loop + * and use the queue where we found work. But with a small + * probability, we'll keep looking for lower priority work, so that + * we don't ignore our low-priority queues entirely. */ + break; + } + } + } + + if (queue == NULL) + return NULL; + + workqueue_entry_t *work = TOR_TAILQ_FIRST(queue); + TOR_TAILQ_REMOVE(queue, work, next_work); + work->pending = 0; + return work; } /** @@ -217,9 +286,9 @@ worker_thread_main(void *thread_) tor_mutex_acquire(&pool->lock); continue; } - work = TOR_TAILQ_FIRST(&pool->work); - TOR_TAILQ_REMOVE(&pool->work, work, next_work); - work->pending = 0; + work = worker_thread_extract_next_work(thread); + if (BUG(work == NULL)) + break; tor_mutex_release(&pool->lock); /* We run the work function without holding the thread lock. This @@ -268,12 +337,14 @@ queue_reply(replyqueue_t *queue, workqueue_entry_t *work) /** Allocate and start a new worker thread to use state object <b>state</b>, * and send responses to <b>replyqueue</b>. */ static workerthread_t * -workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) +workerthread_new(int32_t lower_priority_chance, + void *state, threadpool_t *pool, replyqueue_t *replyqueue) { workerthread_t *thr = tor_malloc_zero(sizeof(workerthread_t)); thr->state = state; thr->reply_queue = replyqueue; thr->in_pool = pool; + thr->lower_priority_chance = lower_priority_chance; if (spawn_func(worker_thread_main, thr) < 0) { //LCOV_EXCL_START @@ -299,24 +370,34 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) * function's responsibility to free the work object. * * On success, return a workqueue_entry_t object that can be passed to - * workqueue_entry_cancel(). On failure, return NULL. + * workqueue_entry_cancel(). On failure, return NULL. (Failure is not + * currently possible, but callers should check anyway.) + * + * Items are executed in a loose priority order -- each thread will usually + * take from the queued work with the highest prioirity, but will occasionally + * visit lower-priority queues to keep them from starving completely. * - * Note that because each thread has its own work queue, work items may not + * Note that because of priorities and thread behavior, work items may not * be executed strictly in order. */ workqueue_entry_t * -threadpool_queue_work(threadpool_t *pool, - workqueue_reply_t (*fn)(void *, void *), - void (*reply_fn)(void *), - void *arg) +threadpool_queue_work_priority(threadpool_t *pool, + workqueue_priority_t prio, + workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg) { + tor_assert(((int)prio) >= WORKQUEUE_PRIORITY_FIRST && + ((int)prio) <= WORKQUEUE_PRIORITY_LAST); + workqueue_entry_t *ent = workqueue_entry_new(fn, reply_fn, arg); ent->on_pool = pool; ent->pending = 1; + ent->priority = prio; tor_mutex_acquire(&pool->lock); - TOR_TAILQ_INSERT_TAIL(&pool->work, ent, next_work); + TOR_TAILQ_INSERT_TAIL(&pool->work[prio], ent, next_work); tor_cond_signal_one(&pool->condition); @@ -325,6 +406,16 @@ threadpool_queue_work(threadpool_t *pool, return ent; } +/** As threadpool_queue_work_priority(), but assumes WQ_PRI_HIGH */ +workqueue_entry_t * +threadpool_queue_work(threadpool_t *pool, + workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg) +{ + return threadpool_queue_work_priority(pool, WQ_PRI_HIGH, fn, reply_fn, arg); +} + /** * Queue a copy of a work item for every thread in a pool. This can be used, * for example, to tell the threads to update some parameter in their states. @@ -388,6 +479,14 @@ threadpool_queue_update(threadpool_t *pool, /** Don't have more than this many threads per pool. */ #define MAX_THREADS 1024 +/** For half of our threads, choose lower priority queues with probability + * 1/N for each of these values. Both are chosen somewhat arbitrarily. If + * CHANCE_PERMISSIVE is too low, then we have a risk of low-priority tasks + * stalling forever. If it's too high, we have a risk of low-priority tasks + * grabbing half of the threads. */ +#define CHANCE_PERMISSIVE 37 +#define CHANCE_STRICT INT32_MAX + /** Launch threads until we have <b>n</b>. */ static int threadpool_start_threads(threadpool_t *pool, int n) @@ -404,8 +503,14 @@ threadpool_start_threads(threadpool_t *pool, int n) sizeof(workerthread_t*), n); while (pool->n_threads < n) { + /* For half of our threads, we'll choose lower priorities permissively; + * for the other half, we'll stick more strictly to higher priorities. + * This keeps slow low-priority tasks from taking over completely. */ + int32_t chance = (pool->n_threads & 1) ? CHANCE_STRICT : CHANCE_PERMISSIVE; + void *state = pool->new_thread_state_fn(pool->new_thread_state_arg); - workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue); + workerthread_t *thr = workerthread_new(chance, + state, pool, pool->reply_queue); if (!thr) { //LCOV_EXCL_START @@ -441,7 +546,15 @@ threadpool_new(int n_threads, pool = tor_malloc_zero(sizeof(threadpool_t)); tor_mutex_init_nonrecursive(&pool->lock); tor_cond_init(&pool->condition); - TOR_TAILQ_INIT(&pool->work); + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + TOR_TAILQ_INIT(&pool->work[i]); + } + { + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(&pool->weak_rng, seed); + } pool->new_thread_state_fn = new_thread_state_fn; pool->new_thread_state_arg = arg; @@ -491,15 +604,41 @@ replyqueue_new(uint32_t alertsocks_flags) return rq; } -/** - * Return the "read socket" for a given reply queue. The main thread should - * listen for read events on this socket, and call replyqueue_process() every - * time it triggers. +/** Internal: Run from the libevent mainloop when there is work to handle in + * the reply queue handler. */ +static void +reply_event_cb(evutil_socket_t sock, short events, void *arg) +{ + threadpool_t *tp = arg; + (void) sock; + (void) events; + replyqueue_process(tp->reply_queue); + if (tp->reply_cb) + tp->reply_cb(tp); +} + +/** Register the threadpool <b>tp</b>'s reply queue with the libevent + * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after + * each time there is work to process from the reply queue. Return 0 on + * success, -1 on failure. */ -tor_socket_t -replyqueue_get_socket(replyqueue_t *rq) +int +threadpool_register_reply_event(threadpool_t *tp, + void (*cb)(threadpool_t *tp)) { - return rq->alert.read_fd; + struct event_base *base = tor_libevent_get_base(); + + if (tp->reply_event) { + tor_event_free(tp->reply_event); + } + tp->reply_event = tor_event_new(base, + tp->reply_queue->alert.read_fd, + EV_READ|EV_PERSIST, + reply_event_cb, + tp); + tor_assert(tp->reply_event); + tp->reply_cb = cb; + return event_add(tp->reply_event, NULL); } /** @@ -510,12 +649,13 @@ replyqueue_get_socket(replyqueue_t *rq) void replyqueue_process(replyqueue_t *queue) { - if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { + int r = queue->alert.drain_fn(queue->alert.read_fd); + if (r < 0) { //LCOV_EXCL_START static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, "Failure from drain_fd: %s", - tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); + tor_socket_strerror(-r)); //LCOV_EXCL_STOP } diff --git a/src/common/workqueue.h b/src/common/workqueue.h index 54276767b0..e1fe612e2b 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_WORKQUEUE_H @@ -16,12 +16,26 @@ typedef struct threadpool_s threadpool_t; typedef struct workqueue_entry_s workqueue_entry_t; /** Possible return value from a work function: */ -typedef enum { +typedef enum workqueue_reply_t { WQ_RPL_REPLY = 0, /** indicates success */ WQ_RPL_ERROR = 1, /** indicates fatal error */ WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */ } workqueue_reply_t; +/** Possible priorities for work. Lower numeric values are more important. */ +typedef enum workqueue_priority_t { + WQ_PRI_HIGH = 0, + WQ_PRI_MED = 1, + WQ_PRI_LOW = 2, +} workqueue_priority_t; + +workqueue_entry_t *threadpool_queue_work_priority(threadpool_t *pool, + workqueue_priority_t prio, + workqueue_reply_t (*fn)(void *, + void *), + void (*reply_fn)(void *), + void *arg); + workqueue_entry_t *threadpool_queue_work(threadpool_t *pool, workqueue_reply_t (*fn)(void *, void *), @@ -42,8 +56,11 @@ threadpool_t *threadpool_new(int n_threads, replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); -tor_socket_t replyqueue_get_socket(replyqueue_t *rq); void replyqueue_process(replyqueue_t *queue); -#endif +struct event_base; +int threadpool_register_reply_event(threadpool_t *tp, + void (*cb)(threadpool_t *tp)); + +#endif /* !defined(TOR_WORKQUEUE_H) */ |