diff options
Diffstat (limited to 'src')
69 files changed, 4826 insertions, 857 deletions
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index b8c5dd4fea..a1c819fffa 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \ - crypto_curve25519.obj curve25519-donna.obj +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \ + tortls.obj crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj diff --git a/src/common/address.c b/src/common/address.c index 701daace57..c5d0e918ca 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1198,7 +1198,7 @@ tor_addr_hash(const tor_addr_t *addr) /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; - /* LCOV_EXCL_END */ + /* LCOV_EXCL_STOP */ } } diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index 7b28a9494a..c593e9af8d 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -352,3 +352,49 @@ alert_sockets_close(alert_sockets_t *socks) socks->read_fd = socks->write_fd = -1; } +/* + * XXXX We might be smart to move to compiler intrinsics or real atomic + * XXXX operations at some point. But not yet. + * + */ + +/** Initialize a new atomic counter with the value 0 */ +void +atomic_counter_init(atomic_counter_t *counter) +{ + memset(counter, 0, sizeof(*counter)); + tor_mutex_init_nonrecursive(&counter->mutex); +} +/** Clean up all resources held by an atomic counter. */ +void +atomic_counter_destroy(atomic_counter_t *counter) +{ + tor_mutex_uninit(&counter->mutex); + memset(counter, 0, sizeof(*counter)); +} +/** Add a value to an atomic counter. */ +void +atomic_counter_add(atomic_counter_t *counter, size_t add) +{ + tor_mutex_acquire(&counter->mutex); + counter->val += add; + tor_mutex_release(&counter->mutex); +} +/** Subtract a value from an atomic counter. */ +void +atomic_counter_sub(atomic_counter_t *counter, size_t sub) +{ + // this relies on unsigned overflow, but that's fine. + atomic_counter_add(counter, -sub); +} +/** Return the current value of an atomic counter */ +size_t +atomic_counter_get(atomic_counter_t *counter) +{ + size_t val; + tor_mutex_acquire(&counter->mutex); + val = counter->val; + tor_mutex_release(&counter->mutex); + return val; +} + diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h index 2943573b23..9fa3d0d0b7 100644 --- a/src/common/compat_threads.h +++ b/src/common/compat_threads.h @@ -147,5 +147,19 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal); */ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value); +/** + * Atomic counter type; holds a size_t value. + */ +typedef struct atomic_counter_t { + tor_mutex_t mutex; + size_t val; +} atomic_counter_t; + +void atomic_counter_init(atomic_counter_t *counter); +void atomic_counter_destroy(atomic_counter_t *counter); +void atomic_counter_add(atomic_counter_t *counter, size_t add); +void atomic_counter_sub(atomic_counter_t *counter, size_t sub); +size_t atomic_counter_get(atomic_counter_t *counter); + #endif diff --git a/src/common/compress.c b/src/common/compress.c new file mode 100644 index 0000000000..771f5ab7b6 --- /dev/null +++ b/src/common/compress.c @@ -0,0 +1,554 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.c + * \brief Common compression API. + **/ + +#include "orconfig.h" + +#include <stdlib.h> +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include "torint.h" + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" +#include "compress_zlib.h" +#include "compress_zstd.h" + +/** Total number of bytes allocated for compression state overhead. */ +static atomic_counter_t total_compress_allocation; + +/** @{ */ +/* These macros define the maximum allowable compression factor. Anything of + * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to + * have an uncompression factor (uncompressed size:compressed size ratio) of + * any greater than MAX_UNCOMPRESSION_FACTOR. + * + * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to + * be small to limit the attack multiplier, but we also want it to be large + * enough so that no legitimate document --even ones we might invent in the + * future -- ever compresses by a factor of greater than + * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably + * large range of possible values. IMO, anything over 8 is probably safe; IMO + * anything under 50 is probably sufficient. + */ +#define MAX_UNCOMPRESSION_FACTOR 25 +#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ + +/** Return true if uncompressing an input of size <b>in_size</b> to an input of + * size at least <b>size_out</b> looks like a compression bomb. */ +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out) +{ + if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) + return 0; + + return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); +} + +/** Guess the size that <b>in_len</b> will be after compression or + * decompression. */ +static size_t +guess_compress_size(int compress, compress_method_t method, + compression_level_t compression_level, + size_t in_len) +{ + // ignore these for now. + (void)method; + (void)compression_level; + + /* Always guess a factor of 2. */ + if (compress) { + in_len /= 2; + } else { + if (in_len < SIZE_T_CEILING/2) + in_len *= 2; + } + return MAX(in_len, 1024); +} + +/** Internal function to implement tor_compress/tor_uncompress, depending on + * whether <b>compress</b> is set. All arguments are as for tor_compress or + * tor_uncompress. */ +static int +tor_compress_impl(int compress, + char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + compression_level_t compression_level, + int complete_only, + int protocol_warn_level) +{ + tor_compress_state_t *stream; + int rv; + + stream = tor_compress_new(compress, method, compression_level); + + if (stream == NULL) + return -1; + + size_t in_len_orig = in_len; + size_t out_remaining, out_alloc; + char *outptr; + + out_remaining = out_alloc = + guess_compress_size(compress, method, compression_level, in_len); + *out = outptr = tor_malloc(out_remaining); + + const int finish = complete_only || compress; + + while (1) { + switch (tor_compress_process(stream, + &outptr, &out_remaining, + &in, &in_len, finish)) { + case TOR_COMPRESS_DONE: + if (in_len == 0 || compress) { + goto done; + } else { + // More data is present, and we're decompressing. So we may need to + // reinitialize the stream if we are handling multiple concatenated + // inputs. + tor_compress_free(stream); + stream = tor_compress_new(compress, method, compression_level); + } + break; + case TOR_COMPRESS_OK: + if (compress || complete_only) { + goto err; + } else { + goto done; + } + break; + case TOR_COMPRESS_BUFFER_FULL: { + if (!compress && outptr < *out+out_alloc) { + // A buffer error in this case means that we have a problem + // with our input. + log_fn(protocol_warn_level, LD_PROTOCOL, + "Possible truncated or corrupt compressed data"); + goto err; + } + if (out_alloc >= SIZE_T_CEILING / 2) { + log_warn(LD_GENERAL, "While %scompresing data: ran out of space.", + compress?"":"un"); + goto err; + } + if (!compress && + tor_compress_is_compression_bomb(in_len_orig, out_alloc)) { + // This should already have been caught down in the backend logic. + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + const size_t offset = outptr - *out; + out_alloc *= 2; + *out = tor_realloc(*out, out_alloc); + outptr = *out + offset; + out_remaining = out_alloc - offset; + break; + } + case TOR_COMPRESS_ERROR: + log_fn(protocol_warn_level, LD_GENERAL, + "Error while %scompresing data: bad input?", + compress?"":"un"); + goto err; // bad data. + default: + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + } + done: + *out_len = outptr - *out; + if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a " + "compression bomb."); + goto err; + } + if (!compress) { + // NUL-terminate our output. + if (out_alloc == *out_len) + *out = tor_realloc(*out, out_alloc + 1); + (*out)[*out_len] = '\0'; + } + rv = 0; + goto out; + + err: + tor_free(*out); + *out_len = 0; + rv = -1; + goto out; + + out: + tor_compress_free(stream); + return rv; +} + +/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly + * allocated buffer, using the method described in <b>method</b>. Store the + * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. + * Return 0 on success, -1 on failure. + */ +int +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + return tor_compress_impl(1, out, out_len, in, in_len, method, + BEST_COMPRESSION, + 1, LOG_WARN); +} + +/** Given zero or more compressed strings of total length <b>in_len</b> bytes + * at <b>in</b>, uncompress them into a newly allocated buffer, using the + * method described in <b>method</b>. Store the uncompressed string in + * *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 on + * failure. + * + * If <b>complete_only</b> is true, we consider a truncated input as a + * failure; otherwise we decompress as much as we can. Warn about truncated + * or corrupt inputs at <b>protocol_warn_level</b>. + */ +int +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + return tor_compress_impl(0, out, out_len, in, in_len, method, + BEST_COMPRESSION, + complete_only, protocol_warn_level); +} + +/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely + * to be compressed or not. If it is, return the likeliest compression method. + * Otherwise, return UNKNOWN_METHOD. + */ +compress_method_t +detect_compression_method(const char *in, size_t in_len) +{ + if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { + return GZIP_METHOD; + } else if (in_len > 2 && (in[0] & 0x0f) == 8 && + (ntohs(get_uint16(in)) % 31) == 0) { + return ZLIB_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x5d\x00\x00\x00", 4)) { + return LZMA_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) { + return ZSTD_METHOD; + } else { + return UNKNOWN_METHOD; + } +} + +/** Return 1 if a given <b>method</b> is supported; otherwise 0. */ +int +tor_compress_supports_method(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_method_supported(); + case LZMA_METHOD: + return tor_lzma_method_supported(); + case ZSTD_METHOD: + return tor_zstd_method_supported(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return 0; + } +} + +/** Table of compression method names. These should have an "x-" prefix, + * if they are not listed in the IANA content coding registry. */ +static const struct { + const char *name; + compress_method_t method; +} compression_method_names[] = { + { "gzip", GZIP_METHOD }, + { "deflate", ZLIB_METHOD }, + { "x-lzma", LZMA_METHOD }, + { "x-zstd" , ZSTD_METHOD }, + { "identity", NO_METHOD }, + + /* Later entries in this table are not canonical; these are recognized but + * not emitted. */ + { "x-gzip", GZIP_METHOD }, +}; + +/** Return the canonical string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (method == compression_method_names[i].method) + return compression_method_names[i].name; + } + return NULL; +} + +/** Return the compression method represented by the string <b>name</b>, or + * UNKNOWN_METHOD if the string isn't recognized. */ +compress_method_t +compression_method_get_by_name(const char *name) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (!strcmp(compression_method_names[i].name, name)) + return compression_method_names[i].method; + } + return UNKNOWN_METHOD; +} + +/** Return a string representation of the version of the library providing the + * compression method given in <b>method</b>. Returns NULL if <b>method</b> is + * unknown or unsupported. */ +const char * +tor_compress_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_version_str(); + case LZMA_METHOD: + return tor_lzma_get_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return a string representation of the version of the library, found at + * compile time, providing the compression method given in <b>method</b>. + * Returns NULL if <b>method</b> is unknown or unsupported. */ +const char * +tor_compress_header_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_header_version_str(); + case LZMA_METHOD: + return tor_lzma_get_header_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_header_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return the approximate number of bytes allocated for all + * supported compression schemas. */ +size_t +tor_compress_get_total_allocation(void) +{ + return atomic_counter_get(&total_compress_allocation) + + tor_zlib_get_total_allocation() + + tor_lzma_get_total_allocation() + + tor_zstd_get_total_allocation(); +} + +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { + compress_method_t method; /**< The compression method. */ + + union { + tor_zlib_compress_state_t *zlib_state; + tor_lzma_compress_state_t *lzma_state; + tor_zstd_compress_state_t *zstd_state; + } u; /**< Compression backend state. */ +}; + +/** Construct and return a tor_compress_state_t object using <b>method</b>. If + * <b>compress</b>, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress, compress_method_t method, + compression_level_t compression_level) +{ + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + break; + } + case LZMA_METHOD: { + tor_lzma_compress_state_t *lzma_state = + tor_lzma_compress_new(compress, method, compression_level); + + if (lzma_state == NULL) + goto err; + + state->u.lzma_state = lzma_state; + break; + } + case ZSTD_METHOD: { + tor_zstd_compress_state_t *zstd_state = + tor_zstd_compress_new(compress, method, compression_level); + + if (zstd_state == NULL) + goto err; + + state->u.zstd_state = zstd_state; + break; + } + case NO_METHOD: + case UNKNOWN_METHOD: + goto err; + } + + atomic_counter_add(&total_compress_allocation, + sizeof(tor_compress_state_t)); + return state; + + err: + tor_free(state); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + tor_assert(state != NULL); + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + case LZMA_METHOD: + return tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); + case ZSTD_METHOD: + return tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); + case NO_METHOD: + case UNKNOWN_METHOD: + goto err; + } + + err: + return TOR_COMPRESS_ERROR; +} + +/** Deallocate <b>state</b>. */ +void +tor_compress_free(tor_compress_state_t *state) +{ + if (state == NULL) + return; + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + tor_zlib_compress_free(state->u.zlib_state); + break; + case LZMA_METHOD: + tor_lzma_compress_free(state->u.lzma_state); + break; + case ZSTD_METHOD: + tor_zstd_compress_free(state->u.zstd_state); + break; + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } + + atomic_counter_sub(&total_compress_allocation, + sizeof(tor_compress_state_t)); + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_compress_state_size(const tor_compress_state_t *state) +{ + tor_assert(state != NULL); + + size_t size = sizeof(tor_compress_state_t); + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + size += tor_zlib_compress_state_size(state->u.zlib_state); + break; + case LZMA_METHOD: + size += tor_lzma_compress_state_size(state->u.lzma_state); + break; + case ZSTD_METHOD: + size += tor_zstd_compress_state_size(state->u.zstd_state); + break; + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } + + return size; +} + +/** Initialize all compression modules. */ +void +tor_compress_init(void) +{ + atomic_counter_init(&total_compress_allocation); + + tor_zlib_init(); + tor_lzma_init(); + tor_zstd_init(); +} + diff --git a/src/common/compress.h b/src/common/compress.h new file mode 100644 index 0000000000..95b70c02ec --- /dev/null +++ b/src/common/compress.h @@ -0,0 +1,87 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.h + * \brief Headers for compress.c + **/ + +#ifndef TOR_COMPRESS_H +#define TOR_COMPRESS_H + +/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and + * GZIP_METHOD is guaranteed to be supported by the compress/uncompress + * functions here. Call tor_compress_supports_method() to check if a given + * compression schema is supported by Tor. */ +typedef enum { + NO_METHOD=0, + GZIP_METHOD=1, + ZLIB_METHOD=2, + LZMA_METHOD=3, + ZSTD_METHOD=4, + UNKNOWN_METHOD=5 +} compress_method_t; + +/** + * Enumeration to define tradeoffs between memory usage and compression level. + * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most + * memory. + **/ +typedef enum { + BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION +} compression_level_t; + +int tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +compress_method_t detect_compression_method(const char *in, size_t in_len); + +int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); + +int tor_compress_supports_method(compress_method_t method); +const char * compression_method_get_name(compress_method_t method); +compress_method_t compression_method_get_by_name(const char *name); + +const char *tor_compress_version_str(compress_method_t method); + +const char *tor_compress_header_version_str(compress_method_t method); + +size_t tor_compress_get_total_allocation(void); + +/** Return values from tor_compress_process; see that function's documentation + * for details. */ +typedef enum { + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; + +/** Internal state for an incremental compression/decompression. */ +typedef struct tor_compress_state_t tor_compress_state_t; + +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); + +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free(tor_compress_state_t *state); + +size_t tor_compress_state_size(const tor_compress_state_t *state); + +void tor_compress_init(void); + +#endif // TOR_COMPRESS_H. + diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c new file mode 100644 index 0000000000..b721bae8aa --- /dev/null +++ b/src/common/compress_lzma.c @@ -0,0 +1,350 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.c + * \brief Compression backend for LZMA. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" + +#ifdef HAVE_LZMA +#include <lzma.h> +#endif + +/** Total number of bytes allocated for LZMA state. */ +static atomic_counter_t total_lzma_allocation; + +#ifdef HAVE_LZMA +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 6; + case LOW_COMPRESSION: return 3; + } +} + +/** Convert a given <b>error</b> to a human readable error string. */ +static const char * +lzma_error_str(lzma_ret error) +{ + switch (error) { + case LZMA_OK: + return "Operation completed successfully"; + case LZMA_STREAM_END: + return "End of stream"; + case LZMA_NO_CHECK: + return "Input stream lacks integrity check"; + case LZMA_UNSUPPORTED_CHECK: + return "Unable to calculate integrity check"; + case LZMA_GET_CHECK: + return "Integrity check available"; + case LZMA_MEM_ERROR: + return "Unable to allocate memory"; + case LZMA_MEMLIMIT_ERROR: + return "Memory limit reached"; + case LZMA_FORMAT_ERROR: + return "Unknown file format"; + case LZMA_OPTIONS_ERROR: + return "Unsupported options"; + case LZMA_DATA_ERROR: + return "Corrupt input data"; + case LZMA_BUF_ERROR: + return "Unable to progress"; + case LZMA_PROG_ERROR: + return "Programming error"; + default: + return "Unknown LZMA error"; + } +} +#endif // HAVE_LZMA. + +/** Return 1 if LZMA compression is supported; otherwise 0. */ +int +tor_lzma_method_supported(void) +{ +#ifdef HAVE_LZMA + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of liblzma. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_version_str(void) +{ +#ifdef HAVE_LZMA + return lzma_version_string(); +#else + return NULL; +#endif +} + +/** Return a string representation of the version of liblzma used at + * compilation time. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_header_version_str(void) +{ +#ifdef HAVE_LZMA + return LZMA_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal LZMA state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_lzma_compress_state_t { +#ifdef HAVE_LZMA + lzma_stream stream; /**< The LZMA stream. */ +#endif + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_LZMA +/** Return an approximate number of bytes stored in memory to hold the LZMA + * encoder/decoder state. */ +static size_t +tor_lzma_state_size_precalc(int compress, compression_level_t level) +{ + uint64_t memory_usage; + + if (compress) + memory_usage = lzma_easy_encoder_memusage(memory_level(level)); + else + memory_usage = lzma_easy_decoder_memusage(memory_level(level)); + + if (memory_usage == UINT64_MAX) { + log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s", + compress ? "encoder" : "decoder"); + goto err; + } + + if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX) + memory_usage = SIZE_MAX; + else + memory_usage += sizeof(tor_lzma_compress_state_t); + + return (size_t)memory_usage; + + err: + return 0; +} +#endif // HAVE_LZMA. + +/** Construct and return a tor_lzma_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == LZMA_METHOD); + +#ifdef HAVE_LZMA + tor_lzma_compress_state_t *result; + lzma_ret retval; + lzma_options_lzma stream_options; + + // Note that we do not explicitly initialize the lzma_stream object here, + // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is + // also what `tor_malloc_zero()` does. + result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); + result->compress = compress; + result->allocation = tor_lzma_state_size_precalc(compress, level); + + if (compress) { + lzma_lzma_preset(&stream_options, memory_level(level)); + + retval = lzma_alone_encoder(&result->stream, &stream_options); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } else { + // FIXME(ahf): This should be something more sensible than + // UINT64_MAX: See #21665. + retval = lzma_alone_decoder(&result->stream, UINT64_MAX); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } + + atomic_counter_add(&total_lzma_allocation, result->allocation); + return result; + + err: + tor_free(result); + return NULL; +#else // HAVE_LZMA. + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif // HAVE_LZMA. +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_LZMA + lzma_ret retval; + lzma_action action; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + state->stream.next_in = (unsigned char *)*in; + state->stream.avail_in = *in_len; + state->stream.next_out = (unsigned char *)*out; + state->stream.avail_out = *out_len; + + action = finish ? LZMA_FINISH : LZMA_RUN; + + retval = lzma_code(&state->stream, action); + + state->input_so_far += state->stream.next_in - ((unsigned char *)*in); + state->output_so_far += state->stream.next_out - ((unsigned char *)*out); + + *out = (char *)state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *)state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (retval) { + case LZMA_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_OK; + + case LZMA_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + + return TOR_COMPRESS_BUFFER_FULL; + + case LZMA_STREAM_END: + return TOR_COMPRESS_DONE; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + lzma_error_str(retval)); + return TOR_COMPRESS_ERROR; + } +#else // HAVE_LZMA. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + return TOR_COMPRESS_ERROR; +#endif // HAVE_LZMA. +} + +/** Deallocate <b>state</b>. */ +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_lzma_allocation, state->allocation); + +#ifdef HAVE_LZMA + lzma_end(&state->stream); +#endif + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all LZMA states. */ +size_t +tor_lzma_get_total_allocation(void) +{ + return atomic_counter_get(&total_lzma_allocation); +} + +/** Initialize the lzma module */ +void +tor_lzma_init(void) +{ + atomic_counter_init(&total_lzma_allocation); +} + diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h new file mode 100644 index 0000000000..1433c89f88 --- /dev/null +++ b/src/common/compress_lzma.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.h + * \brief Header for compress_lzma.c + **/ + +#ifndef TOR_COMPRESS_LZMA_H +#define TOR_COMPRESS_LZMA_H + +int tor_lzma_method_supported(void); + +const char *tor_lzma_get_version_str(void); + +const char *tor_lzma_get_header_version_str(void); + +/** Internal state for an incremental LZMA compression/decompression. */ +typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; + +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_lzma_compress_free(tor_lzma_compress_state_t *state); + +size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); + +size_t tor_lzma_get_total_allocation(void); + +void tor_lzma_init(void); + +#endif // TOR_COMPRESS_LZMA_H. + diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000000..284542e885 --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,304 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include <zlib.h> + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static atomic_counter_t total_zlib_allocation; + +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: return 9; + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + +/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return 1 if zlib/gzip compression is supported; otherwise 0. */ +int +tor_zlib_method_supported(void) +{ + /* We currently always support zlib/gzip, but we keep this function around in + * case we some day decide to deprecate zlib/gzip support. + */ + return 1; +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits <b>windowBits</b> and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress_, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress_) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = BEST_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress_; + bits = method_bits(method, compression_level); + memlevel = memory_level(compression_level); + if (compress_) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); + + atomic_counter_add(&total_zlib_allocation, out->allocation); + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + if (*in_len > UINT_MAX || + *out_len > UINT_MAX) { + return TOR_COMPRESS_ERROR; + } + + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : "<no message>"); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate <b>state</b>. */ +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zlib_allocation, state->allocation); + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return atomic_counter_get(&total_zlib_allocation); +} + +/** Set up global state for the zlib module */ +void +tor_zlib_init(void) +{ + atomic_counter_init(&total_zlib_allocation); +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000000..df5c196ac7 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +int tor_zlib_method_supported(void); + +const char *tor_zlib_get_version_str(void); + +const char *tor_zlib_get_header_version_str(void); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zlib_compress_free(tor_zlib_compress_state_t *state); + +size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t tor_zlib_get_total_allocation(void); + +void tor_zlib_init(void); + +#endif // TOR_COMPRESS_ZLIB_H. + diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c new file mode 100644 index 0000000000..9c134fdf4e --- /dev/null +++ b/src/common/compress_zstd.c @@ -0,0 +1,393 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.c + * \brief Compression backend for Zstandard. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zstd.h" + +#ifdef HAVE_ZSTD +#include <zstd.h> +#endif + +/** Total number of bytes allocated for Zstandard state. */ +static atomic_counter_t total_zstd_allocation; + +#ifdef HAVE_ZSTD +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 8; + case LOW_COMPRESSION: return 7; + } +} +#endif // HAVE_ZSTD. + +/** Return 1 if Zstandard compression is supported; otherwise 0. */ +int +tor_zstd_method_supported(void) +{ +#ifdef HAVE_ZSTD + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of libzstd. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_version_str(void) +{ +#ifdef HAVE_ZSTD + static char version_str[16]; + size_t version_number; + + version_number = ZSTD_versionNumber(); + tor_snprintf(version_str, sizeof(version_str), + "%lu.%lu.%lu", + version_number / 10000 % 100, + version_number / 100 % 100, + version_number % 100); + + return version_str; +#else + return NULL; +#endif +} + +/** Return a string representation of the version of the version of libzstd + * used at compilation time. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_header_version_str(void) +{ +#ifdef HAVE_ZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal Zstandard state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zstd_compress_state_t { +#ifdef HAVE_ZSTD + union { + /** Compression stream. Used when <b>compress</b> is true. */ + ZSTD_CStream *compress_stream; + /** Decompression stream. Used when <b>compress</b> is false. */ + ZSTD_DStream *decompress_stream; + } u; /**< Zstandard stream objects. */ +#endif // HAVE_ZSTD. + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_ZSTD +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ + tor_assert(preset > 0); + + size_t memory_usage = sizeof(tor_zstd_compress_state_t); + + // The Zstandard library provides a number of functions that would be useful + // here, but they are, unfortunately, still considered experimental and are + // thus only available in libzstd if we link against the library statically. + // + // The code in this function tries to approximate the calculations without + // being able to use the following: + // + // - We do not have access to neither the internal members of ZSTD_CStream + // and ZSTD_DStream and their internal context objects. + // + // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they + // are unexposed. + // + // In the future it might be useful to check if libzstd have started + // providing these functions in a stable manner and simplify this function. + if (compress) { + // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine: + memory_usage += 192; + + // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to + // variables that are not exposed via the public API. We use a _very_ + // simplified function to calculate the estimated amount of bytes used in + // this struct. + // memory_usage += (preset - 0.5) * 1024 * 1024; + memory_usage += (preset * 1024 * 1024) - (512 * 1024); + // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes. + // - stream->outBuffSize: 128 KB: + memory_usage += 128 * 1024; + // - stream->inBuffSize: 2048 KB: + memory_usage += 2048 * 1024; + } else { + // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine: + memory_usage += 208; + // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB. + memory_usage += 150 * 1024; + + // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes. + // - stream->inBuffSize: 0 KB. + // - stream->outBuffSize: 0 KB. + } + + return memory_usage; +} +#endif // HAVE_ZSTD. + +/** Construct and return a tor_zstd_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == ZSTD_METHOD); + +#ifdef HAVE_ZSTD + const int preset = memory_level(level); + tor_zstd_compress_state_t *result; + size_t retval; + + result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); + result->compress = compress; + result->allocation = tor_zstd_state_size_precalc(compress, preset); + + if (compress) { + result->u.compress_stream = ZSTD_createCStream(); + + if (result->u.compress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initCStream(result->u.compress_stream, preset); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } else { + result->u.decompress_stream = ZSTD_createDStream(); + + if (result->u.decompress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initDStream(result->u.decompress_stream); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } + + atomic_counter_add(&total_zstd_allocation, result->allocation); + return result; + + err: + if (compress) { + ZSTD_freeCStream(result->u.compress_stream); + } else { + ZSTD_freeDStream(result->u.decompress_stream); + } + + tor_free(result); + return NULL; +#else // HAVE_ZSTD. + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif // HAVE_ZSTD. +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_ZSTD + size_t retval; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + ZSTD_inBuffer input = { *in, *in_len, 0 }; + ZSTD_outBuffer output = { *out, *out_len, 0 }; + + if (state->compress) { + retval = ZSTD_compressStream(state->u.compress_stream, + &output, &input); + } else { + retval = ZSTD_decompressStream(state->u.decompress_stream, + &output, &input); + } + + state->input_so_far += input.pos; + state->output_so_far += output.pos; + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + *in = (char *)input.src + input.pos; + *in_len = input.size - input.pos; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (state->compress && !finish) { + retval = ZSTD_flushStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + } + + if (state->compress && finish) { + retval = ZSTD_endStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to write " + "epilogue: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // endStream returns the number of bytes that is needed to write the + // epilogue. + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + } + + return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK; +#else // HAVE_ZSTD. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + + return TOR_COMPRESS_ERROR; +#endif // HAVE_ZSTD. +} + +/** Deallocate <b>state</b>. */ +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zstd_allocation, state->allocation); + +#ifdef HAVE_ZSTD + if (state->compress) { + ZSTD_freeCStream(state->u.compress_stream); + } else { + ZSTD_freeDStream(state->u.decompress_stream); + } +#endif // HAVE_ZSTD. + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all Zstandard + * states. */ +size_t +tor_zstd_get_total_allocation(void) +{ + return atomic_counter_get(&total_zstd_allocation); +} + +/** Initialize the zstd module */ +void +tor_zstd_init(void) +{ + atomic_counter_init(&total_zstd_allocation); +} + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h new file mode 100644 index 0000000000..d3e65c2f16 --- /dev/null +++ b/src/common/compress_zstd.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.h + * \brief Header for compress_zstd.c + **/ + +#ifndef TOR_COMPRESS_ZSTD_H +#define TOR_COMPRESS_ZSTD_H + +int tor_zstd_method_supported(void); + +const char *tor_zstd_get_version_str(void); + +const char *tor_zstd_get_header_version_str(void); + +/** Internal state for an incremental Zstandard compression/decompression. */ +typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; + +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zstd_compress_free(tor_zstd_compress_state_t *state); + +size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); + +size_t tor_zstd_get_total_allocation(void); + +void tor_zstd_init(void); + +#endif // TOR_COMPRESS_ZSTD_H. + diff --git a/src/common/confline.c b/src/common/confline.c index 36f4c875f3..d4468f80ea 100644 --- a/src/common/confline.c +++ b/src/common/confline.c @@ -30,6 +30,24 @@ config_line_append(config_line_t **lst, (*lst) = newline; } +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * and prepend it to *<b>lst</b> */ +void +config_line_prepend(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = *lst; + *lst = newline; +} + /** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or * NULL if no such key exists. * diff --git a/src/common/confline.h b/src/common/confline.h index c695bf6b23..477c6929a2 100644 --- a/src/common/confline.h +++ b/src/common/confline.h @@ -31,6 +31,8 @@ typedef struct config_line_t { void config_line_append(config_line_t **lst, const char *key, const char *val); +void config_line_prepend(config_line_t **lst, + const char *key, const char *val); config_line_t *config_lines_dup(const config_line_t *inp); config_line_t *config_lines_dup_and_filter(const config_line_t *inp, const char *key); diff --git a/src/common/include.am b/src/common/include.am index d6504c7466..e285ef5f86 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -105,11 +105,14 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress.c \ + src/common/compress_lzma.c \ + src/common/compress_zlib.c \ + src/common/compress_zstd.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ src/common/crypto_format.c \ - src/common/torgzip.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -145,6 +148,10 @@ COMMONHEADERS = \ src/common/compat_openssl.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress.h \ + src/common/compress_lzma.h \ + src/common/compress_zlib.h \ + src/common/compress_zstd.h \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ @@ -163,7 +170,6 @@ COMMONHEADERS = \ src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ - src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/torgzip.c b/src/common/torgzip.c deleted file mode 100644 index 223cd307a7..0000000000 --- a/src/common/torgzip.c +++ /dev/null @@ -1,584 +0,0 @@ -/* Copyright (c) 2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2017, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.c - * \brief A simple in-memory gzip implementation. - **/ - -#include "orconfig.h" - -#include <stdlib.h> -#include <stdio.h> -#include <assert.h> -#include <string.h> -#include "torint.h" - -#ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> -#endif - -#include "util.h" -#include "torlog.h" -#include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include <zlib.h> - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ -static inline int -method_bits(compress_method_t method, zlib_compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - -static inline int -get_memlevel(zlib_compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - -/** @{ */ -/* These macros define the maximum allowable compression factor. Anything of - * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to - * have an uncompression factor (uncompressed size:compressed size ratio) of - * any greater than MAX_UNCOMPRESSION_FACTOR. - * - * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to - * be small to limit the attack multiplier, but we also want it to be large - * enough so that no legitimate document --even ones we might invent in the - * future -- ever compresses by a factor of greater than - * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably - * large range of possible values. IMO, anything over 8 is probably safe; IMO - * anything under 50 is probably sufficient. - */ -#define MAX_UNCOMPRESSION_FACTOR 25 -#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) -/** @} */ - -/** Return true if uncompressing an input of size <b>in_size</b> to an input - * of size at least <b>size_out</b> looks like a compression bomb. */ -static int -is_compression_bomb(size_t size_in, size_t size_out) -{ - if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) - return 0; - - return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); -} - -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the method described in <b>method</b>. Store the - * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - */ -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#if defined(OpenBSD) - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); - return -1; -} - -/** Given zero or more zlib-compressed or gzip-compressed strings of - * total length - * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated - * buffer, using the method described in <b>method</b>. Store the uncompressed - * string in *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on - * success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a - * failure; otherwise we decompress as much as we can. Warn about truncated - * or corrupt inputs at <b>protocol_warn_level</b>. - */ -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } - return -1; -} - -/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely - * to be compressed or not. If it is, return the likeliest compression method. - * Otherwise, return UNKNOWN_METHOD. - */ -compress_method_t -detect_compression_method(const char *in, size_t in_len) -{ - if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { - return GZIP_METHOD; - } else if (in_len > 2 && (in[0] & 0x0f) == 8 && - (ntohs(get_uint16(in)) % 31) == 0) { - return ZLIB_METHOD; - } else { - return UNKNOWN_METHOD; - } -} - -/** Internal state for an incremental zlib compression/decompression. The - * body of this struct is not exposed. */ -struct tor_zlib_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ - - /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; -}; - -/** Construct and return a tor_zlib_state_t object using <b>method</b>. If - * <b>compress</b>, it's for compression; otherwise it's for - * decompression. */ -tor_zlib_state_t * -tor_zlib_new(int compress_, compress_method_t method, - zlib_compression_level_t compression_level) -{ - tor_zlib_state_t *out; - int bits, memlevel; - - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } - - out = tor_malloc_zero(sizeof(tor_zlib_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - - total_zlib_allocation += out->allocation; - - return out; - - err: - tor_free(out); - return NULL; -} - -/** Compress/decompress some bytes using <b>state</b>. Read up to - * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes - * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, - * we've reached the end of the input. - * - * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. - * Return TOR_ZLIB_OK if we're processed everything from the input. - * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>. - * Return TOR_ZLIB_ERR if the stream is corrupt. - */ -tor_zlib_output_t -tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) -{ - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; - - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } - - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); - - *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; - - if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_ZLIB_ERR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_ZLIB_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_ZLIB_OK; - return TOR_ZLIB_BUF_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_ZLIB_BUF_FULL; - return TOR_ZLIB_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_ZLIB_ERR; - } -} - -/** Deallocate <b>state</b>. */ -void -tor_zlib_free(tor_zlib_state_t *state) -{ - if (!state) - return; - - total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); - - tor_free(state); -} - -/** Return an approximate number of bytes used in RAM to hold a state with - * window bits <b>windowBits</b> and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - -/** Return the approximate number of bytes allocated for <b>state</b>. */ -size_t -tor_zlib_state_size(const tor_zlib_state_t *state) -{ - return state->allocation; -} - -/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; -} - diff --git a/src/common/torgzip.h b/src/common/torgzip.h deleted file mode 100644 index 430b152fe4..0000000000 --- a/src/common/torgzip.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2017, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.h - * \brief Headers for torgzip.h - **/ - -#ifndef TOR_TORGZIP_H -#define TOR_TORGZIP_H - -/** Enumeration of what kind of compression to use. Only ZLIB_METHOD is - * guaranteed to be supported by the compress/uncompress functions here; - * GZIP_METHOD may be supported if we built against zlib version 1.2 or later - * and is_gzip_supported() returns true. */ -typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 -} compress_method_t; - -/** - * Enumeration to define tradeoffs between memory usage and compression level. - * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most - * memory. - **/ -typedef enum { - HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} zlib_compression_level_t; - -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - -int is_gzip_supported(void); - -const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - -compress_method_t detect_compression_method(const char *in, size_t in_len); - -/** Return values from tor_zlib_process; see that function's documentation for - * details. */ -typedef enum { - TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR -} tor_zlib_output_t; -/** Internal state for an incremental zlib compression/decompression. */ -typedef struct tor_zlib_state_t tor_zlib_state_t; -tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - zlib_compression_level_t level); - -tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_zlib_free(tor_zlib_state_t *state); - -size_t tor_zlib_state_size(const tor_zlib_state_t *state); -size_t tor_zlib_get_total_allocation(void); - -#endif - diff --git a/src/common/workqueue.h b/src/common/workqueue.h index 5d14e7f6a7..7b483eb7ac 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -16,7 +16,7 @@ typedef struct threadpool_s threadpool_t; typedef struct workqueue_entry_s workqueue_entry_t; /** Possible return value from a work function: */ -typedef enum { +typedef enum workqueue_reply_t { WQ_RPL_REPLY = 0, /** indicates success */ WQ_RPL_ERROR = 1, /** indicates fatal error */ WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */ diff --git a/src/include.am b/src/include.am index d12684e187..f78853f50f 100644 --- a/src/include.am +++ b/src/include.am @@ -7,3 +7,4 @@ include src/tools/include.am include src/win32/include.am include src/config/include.am include src/test/fuzz/include.am +include src/trace/include.am diff --git a/src/or/buffers.c b/src/or/buffers.c index e559f80a1e..4f22935d26 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -2088,13 +2088,13 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) } /** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the - * zlib state <b>state</b>, appending the result to <b>buf</b>. If + * compression state <b>state</b>, appending the result to <b>buf</b>. If * <b>done</b> is true, flush the data in the state and finish the * compression/uncompression. Return -1 on failure, 0 on success. */ int -write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, - int done) +write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + int done) { char *next; size_t old_avail, avail; @@ -2108,20 +2108,22 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, } next = CHUNK_WRITE_PTR(buf->tail); avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); - switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) { - case TOR_ZLIB_DONE: + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: over = 1; break; - case TOR_ZLIB_ERR: + case TOR_COMPRESS_ERROR: return -1; - case TOR_ZLIB_OK: + case TOR_COMPRESS_OK: if (data_len == 0) over = 1; break; - case TOR_ZLIB_BUF_FULL: + case TOR_COMPRESS_BUFFER_FULL: if (avail) { - /* Zlib says we need more room (ZLIB_BUF_FULL). Start a new chunk - * automatically, whether were going to or not. */ + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ need_new_chunk = 1; } break; diff --git a/src/or/buffers.h b/src/or/buffers.h index c6a5ffaad5..23b58a571a 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -36,8 +36,8 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, int done); +int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, int done); int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); int fetch_from_buf(char *string, size_t string_len, buf_t *buf); int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 80bb7f69f3..365e5b2f89 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1990,10 +1990,10 @@ single_conn_free_bytes(connection_t *conn) } if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); - if (dir_conn->zlib_state) { - result += tor_zlib_state_size(dir_conn->zlib_state); - tor_zlib_free(dir_conn->zlib_state); - dir_conn->zlib_state = NULL; + if (dir_conn->compress_state) { + result += tor_compress_state_size(dir_conn->compress_state); + tor_compress_free(dir_conn->compress_state); + dir_conn->compress_state = NULL; } } return result; diff --git a/src/or/config.c b/src/or/config.c index 809ff499fc..a73f397073 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -99,7 +100,6 @@ #include "statefile.h" #include "transports.h" #include "ext_orport.h" -#include "torgzip.h" #ifdef _WIN32 #include <shlobj.h> #endif @@ -4949,9 +4949,21 @@ options_init_from_torrc(int argc, char **argv) printf("OpenSSL \t\t%-15s\t\t%s\n", crypto_openssl_get_header_version_str(), crypto_openssl_get_version_str()); - printf("Zlib \t\t%-15s\t\t%s\n", - tor_zlib_get_header_version_str(), - tor_zlib_get_version_str()); + if (tor_compress_supports_method(ZLIB_METHOD)) { + printf("Zlib \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZLIB_METHOD), + tor_compress_header_version_str(ZLIB_METHOD)); + } + if (tor_compress_supports_method(LZMA_METHOD)) { + printf("Liblzma \t\t%-15s\t\t%s\n", + tor_compress_version_str(LZMA_METHOD), + tor_compress_header_version_str(LZMA_METHOD)); + } + if (tor_compress_supports_method(ZSTD_METHOD)) { + printf("Libzstd \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZSTD_METHOD), + tor_compress_header_version_str(ZSTD_METHOD)); + } //TODO: Hex versions? exit(0); } diff --git a/src/or/connection.c b/src/or/connection.c index 09e316d214..5fb2c53677 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -628,7 +628,7 @@ connection_free_(connection_t *conn) dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - tor_zlib_free(dir_conn->zlib_state); + tor_compress_free(dir_conn->compress_state); if (dir_conn->spool) { SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled, spooled_resource_free(spooled)); @@ -4060,9 +4060,9 @@ connection_write_to_buf_impl_,(const char *string, size_t len, if (zlib) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); int done = zlib < 0; - CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf, - dir_conn->zlib_state, - string, len, done)); + CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf, + dir_conn->compress_state, + string, len, done)); } else { CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf)); } diff --git a/src/or/connection.h b/src/or/connection.h index df6fc64709..36e45aef38 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -141,17 +141,17 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); -/* DOCDOC connection_write_to_buf_zlib */ -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); +/* DOCDOC connection_write_to_buf_compress */ +static void connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done); static inline void connection_write_to_buf(const char *string, size_t len, connection_t *conn) { connection_write_to_buf_impl_(string, len, conn, 0); } static inline void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) +connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done) { connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1); } diff --git a/src/or/conscache.c b/src/or/conscache.c index 2a6e1445e3..9dedb43085 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -16,6 +16,7 @@ */ struct consensus_cache_entry_t { uint32_t magic; /**< Must be set to CCE_MAGIC */ + HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t); int32_t refcnt; /**< Reference count. */ unsigned can_remove : 1; /**< If true, we want to delete this file. */ /** If true, we intend to unmap this file as soon as we're done with it. */ @@ -174,6 +175,8 @@ consensus_cache_find_first(consensus_cache_t *cache, * Given a <b>cache</b>, add every entry to <b>out<b> for which * <b>key</b>=<b>value</b>. If <b>key</b> is NULL, add every entry. * + * Do not add any entry that has been marked for removal. + * * Does not adjust reference counts. */ void @@ -182,12 +185,15 @@ consensus_cache_find_all(smartlist_t *out, const char *key, const char *value) { - if (! key) { - smartlist_add_all(out, cache->entries); - return; - } - SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { + if (ent->can_remove == 1) { + /* We want to delete this; pretend it isn't there. */ + continue; + } + if (! key) { + smartlist_add(out, ent); + continue; + } const char *found_val = consensus_cache_entry_get_value(ent, key); if (found_val && !strcmp(value, found_val)) { smartlist_add(out, ent); @@ -300,6 +306,7 @@ consensus_cache_entry_decref(consensus_cache_entry_t *ent) } tor_free(ent->fname); config_free_lines(ent->labels); + consensus_cache_entry_handles_clear(ent); memwipe(ent, 0, sizeof(consensus_cache_entry_t)); tor_free(ent); } @@ -485,6 +492,8 @@ consensus_cache_entry_unmap(consensus_cache_entry_t *ent) ent->unused_since = TIME_MAX; } +HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, ) + #ifdef TOR_UNIT_TESTS /** * Testing only: Return true iff <b>ent</b> is mapped into memory. diff --git a/src/or/conscache.h b/src/or/conscache.h index 94d7f15457..c8cda60e53 100644 --- a/src/or/conscache.h +++ b/src/or/conscache.h @@ -4,9 +4,13 @@ #ifndef TOR_CONSCACHE_H #define TOR_CONSCACHE_H +#include "handles.h" + typedef struct consensus_cache_entry_t consensus_cache_entry_t; typedef struct consensus_cache_t consensus_cache_t; +HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, ) + consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries); void consensus_cache_free(consensus_cache_t *cache); void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff); diff --git a/src/or/consdiff.c b/src/or/consdiff.c index d2a2af1b5f..c0fe979b60 100644 --- a/src/or/consdiff.c +++ b/src/or/consdiff.c @@ -671,7 +671,7 @@ gen_ed_diff(const smartlist_t *cons1, const smartlist_t *cons2, */ i1=len1-1, i2=len2-1; char buf[128]; - while (i1 > 0 || i2 > 0) { + while (i1 >= 0 || i2 >= 0) { int start1x, start2x, end1, end2, added, deleted; @@ -757,6 +757,9 @@ get_linenum(const char **s, int *num_out) { int ok; char *next; + if (!TOR_ISDIGIT(**s)) { + return -1; + } *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next); if (ok && next) { *s = next; @@ -795,6 +798,7 @@ apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, diff_line[diff_cdline->len] = 0; const char *ptr = diff_line; int start = 0, end = 0; + int had_range = 0; if (get_linenum(&ptr, &start) < 0) { log_warn(LD_CONSDIFF, "Could not apply consensus diff because " "an ed command was missing a line number."); @@ -802,6 +806,7 @@ apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, } if (*ptr == ',') { /* Two-item range */ + had_range = 1; ++ptr; if (get_linenum(&ptr, &end) < 0) { log_warn(LD_CONSDIFF, "Could not apply consensus diff because " @@ -850,6 +855,13 @@ apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, goto error_cleanup; } + /* 'a' commands are not allowed to have ranges. */ + if (had_range && action == 'a') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it wanted to add lines after a range."); + goto error_cleanup; + } + /* Add unchanged lines. */ for (; j && j > end; --j) { cdline_t *cons_line = smartlist_get(cons1, j-1); diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c new file mode 100644 index 0000000000..a57edf98fc --- /dev/null +++ b/src/or/consdiffmgr.c @@ -0,0 +1,1120 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file consdiffmsr.c + * + * \brief consensus diff manager functions + * + * This module is run by directory authorities and caches in order + * to remember a number of past consensus documents, and to generate + * and serve the diffs from those documents to the latest consensus. + */ + +#define CONSDIFFMGR_PRIVATE + +#include "or.h" +#include "conscache.h" +#include "consdiff.h" +#include "consdiffmgr.h" +#include "cpuworker.h" +#include "networkstatus.h" +#include "workqueue.h" + +/* XXXX support compression */ + +/** + * Labels to apply to items in the conscache object. + * + * @{ + */ +/* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */ +#define LABEL_DOCTYPE "document-type" +/* The valid-after time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_VALID_AFTER "consensus-valid-after" +/* A hex encoded SHA3 digest of the object after decompression. */ +#define LABEL_SHA3_DIGEST "sha3-digest" +/* The flavor of the consensus or consensuses diff */ +#define LABEL_FLAVOR "consensus-flavor" +/* Diff only: the SHA3 digest of the source consensus. */ +#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest" +/* Diff only: the SHA3 digest of the target consensus. */ +#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest" +/* Diff only: the valid-after date of the source consensus. */ +#define LABEL_FROM_VALID_AFTER "from-valid-after" +/** @} */ + +#define DOCTYPE_CONSENSUS "consensus" +#define DOCTYPE_CONSENSUS_DIFF "consensus-diff" + +/** + * Underlying directory that stores consensuses and consensus diffs. Don't + * use this directly: use cdm_cache_get() instead. + */ +static consensus_cache_t *cons_diff_cache = NULL; +/** + * If true, we have learned at least one new consensus since the + * consensus cache was last up-to-date. + */ +static int cdm_cache_dirty = 0; +/** + * If true, we have scanned the cache to update our hashtable of diffs. + */ +static int cdm_cache_loaded = 0; + +/** + * Possible status values for cdm_diff_t.cdm_diff_status + **/ +typedef enum cdm_diff_status_t { + CDM_DIFF_PRESENT=1, + CDM_DIFF_IN_PROGRESS=2, + CDM_DIFF_ERROR=3, +} cdm_diff_status_t; + +/** Hashtable node used to remember the current status of the diff + * from a given sha3 digest to the current consensus. */ +typedef struct cdm_diff_t { + HT_ENTRY(cdm_diff_t) node; + + /** Consensus flavor for this diff (part of ht key) */ + consensus_flavor_t flavor; + /** SHA3-256 digest of the consensus that this diff is _from_. (part of the + * ht key) */ + uint8_t from_sha3[DIGEST256_LEN]; + + /** One of the CDM_DIFF_* values, depending on whether this diff + * is available, in progress, or impossible to compute. */ + cdm_diff_status_t cdm_diff_status; + /** SHA3-256 digest of the consensus that this diff is _to. */ + uint8_t target_sha3[DIGEST256_LEN]; + /** Handle to the cache entry for this diff, if any. We use a handle here + * to avoid thinking too hard about cache entry lifetime issues. */ + consensus_cache_entry_handle_t *entry; +} cdm_diff_t; + +/** Hashtable mapping flavor and source consensus digest to status. */ +static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER(); + +/** + * Configuration for this module + */ +static consdiff_cfg_t consdiff_cfg = { + /* .cache_max_age_hours = */ 24 * 90, + /* .cache_max_num = */ 1440 +}; + +static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, + consensus_cache_entry_t *diff_to); +static void consdiffmgr_set_cache_flags(void); + +/* ===== + * Hashtable setup + * ===== */ + +/** Helper: hash the key of a cdm_diff_t. */ +static unsigned +cdm_diff_hash(const cdm_diff_t *diff) +{ + uint8_t tmp[DIGEST256_LEN + 1]; + memcpy(tmp, diff->from_sha3, DIGEST256_LEN); + tmp[DIGEST256_LEN] = (uint8_t) diff->flavor; + return (unsigned) siphash24g(tmp, sizeof(tmp)); +} +/** Helper: compare two cdm_diff_t objects for key equality */ +static int +cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2) +{ + return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) && + diff1->flavor == diff2->flavor; +} + +HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq) +HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq, + 0.6, tor_reallocarray, tor_free_) + +/** Release all storage held in <b>diff</b>. */ +static void +cdm_diff_free(cdm_diff_t *diff) +{ + if (!diff) + return; + consensus_cache_entry_handle_free(diff->entry); + tor_free(diff); +} + +/** Create and return a new cdm_diff_t with the given values. Does not + * add it to the hashtable. */ +static cdm_diff_t * +cdm_diff_new(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *target_sha3) +{ + cdm_diff_t *ent; + ent = tor_malloc_zero(sizeof(cdm_diff_t)); + ent->flavor = flav; + memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN); + memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN); + return ent; +} + +/** + * Examine the diff hashtable to see whether we know anything about computing + * a diff of type <b>flav</b> between consensuses with the two provided + * SHA3-256 digests. If a computation is in progress, or if the computation + * has already been tried and failed, return 1. Otherwise, note the + * computation as "in progress" so that we don't reattempt it later, and + * return 0. + */ +static int +cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *target_sha3) +{ + struct cdm_diff_t search, *ent; + memset(&search, 0, sizeof(cdm_diff_t)); + search.flavor = flav; + memcpy(search.from_sha3, from_sha3, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + if (ent) { + tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT); + return 1; + } + ent = cdm_diff_new(flav, from_sha3, target_sha3); + ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS; + HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent); + return 0; +} + +/** + * Update the status of the diff of type <b>flav</b> between consensuses with + * the two provided SHA3-256 digests, so that its status becomes + * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b> + * is NULL, then the old handle (if any) is freed, and replaced with NULL. + */ +static void +cdm_diff_ht_set_status(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *to_sha3, + int status, + consensus_cache_entry_handle_t *handle) +{ + struct cdm_diff_t search, *ent; + memset(&search, 0, sizeof(cdm_diff_t)); + search.flavor = flav; + memcpy(search.from_sha3, from_sha3, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + if (!ent) { + ent = cdm_diff_new(flav, from_sha3, to_sha3); + ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS; + HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent); + } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) { + // This can happen under certain really pathological conditions + // if we decide we don't care about a diff before it is actually + // done computing. + return; + } + + tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS); + + ent->cdm_diff_status = status; + consensus_cache_entry_handle_free(ent->entry); + ent->entry = handle; +} + +/** + * Helper: Remove from the hash table every present (actually computed) diff + * of type <b>flav</b> whose target digest does not match + * <b>unless_target_sha3_matches</b>. + * + * This function is used for the hash table to throw away references to diffs + * that do not lead to the most given consensus of a given flavor. + */ +static void +cdm_diff_ht_purge(consensus_flavor_t flav, + const uint8_t *unless_target_sha3_matches) +{ + cdm_diff_t **diff, **next; + for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) { + cdm_diff_t *this = *diff; + + if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT && + flav == (*diff)->flavor) { + + if (consensus_cache_entry_handle_get((*diff)->entry) == NULL) { + /* the underlying entry has gone away; drop this. */ + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + continue; + } + + if (unless_target_sha3_matches && + fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3, + DIGEST256_LEN)) { + /* target hash doesn't match; drop this. */ + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + continue; + } + } + next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff); + } +} + +/** + * Helper: initialize <b>cons_diff_cache</b>. + */ +static void +cdm_cache_init(void) +{ + unsigned n_entries = consdiff_cfg.cache_max_num * 2; + + tor_assert(cons_diff_cache == NULL); + cons_diff_cache = consensus_cache_open("diff-cache", n_entries); + if (cons_diff_cache == NULL) { + // LCOV_EXCL_START + log_err(LD_FS, "Error: Couldn't open storage for consensus diffs."); + tor_assert_unreached(); + // LCOV_EXCL_STOP + } else { + consdiffmgr_set_cache_flags(); + } + cdm_cache_dirty = 1; + cdm_cache_loaded = 0; +} + +/** + * Helper: return the consensus_cache_t * that backs this manager, + * initializing it if needed. + */ +STATIC consensus_cache_t * +cdm_cache_get(void) +{ + if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) { + cdm_cache_init(); + } + return cons_diff_cache; +} + +/** + * Helper: given a list of labels, prepend the hex-encoded SHA3 digest + * of the <b>bodylen</b>-byte object at <b>body</b> to those labels, + * with LABEL_SHA3_DIGEST as its label. + */ +static void +cdm_labels_prepend_sha3(config_line_t **labels, + const uint8_t *body, + size_t bodylen) +{ + uint8_t sha3_digest[DIGEST256_LEN]; + char hexdigest[HEX_DIGEST256_LEN+1]; + crypto_digest256((char *)sha3_digest, + (const char *)body, bodylen, DIGEST_SHA3_256); + base16_encode(hexdigest, sizeof(hexdigest), + (const char *)sha3_digest, sizeof(sha3_digest)); + + config_line_prepend(labels, LABEL_SHA3_DIGEST, hexdigest); +} + +/** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the + * given label, set <b>digest_out</b> to that value (decoded), and return 0. + * + * Return -1 if there is no such label, and -2 if it is badly formatted. */ +STATIC int +cdm_entry_get_sha3_value(uint8_t *digest_out, + consensus_cache_entry_t *ent, + const char *label) +{ + if (ent == NULL) + return -1; + + const char *hex = consensus_cache_entry_get_value(ent, label); + if (hex == NULL) + return -1; + + int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex)); + if (n != DIGEST256_LEN) + return -2; + else + return 0; +} + +/** + * Helper: look for a consensus with the given <b>flavor</b> and + * <b>valid_after</b> time in the cache. Return that consensus if it's + * present, or NULL if it's missing. + */ +STATIC consensus_cache_entry_t * +cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after) +{ + char formatted_time[ISO_TIME_LEN+1]; + format_iso_time_nospace(formatted_time, valid_after); + const char *flavname = networkstatus_get_flavor_name(flavor); + + /* We'll filter by valid-after time first, since that should + * match the fewest documents. */ + /* We could add an extra hashtable here, but since we only do this scan + * when adding a new consensus, it probably doesn't matter much. */ + smartlist_t *matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_VALID_AFTER, formatted_time); + consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + consensus_cache_entry_t *result = NULL; + if (smartlist_len(matches) > 1) { + log_warn(LD_BUG, "How odd; there appear to be two matching consensuses " + "with flavor %s published at %s.", + flavname, formatted_time); + } + if (smartlist_len(matches)) { + result = smartlist_get(matches, 0); + } + smartlist_free(matches); + + return result; +} + +/** + * Given a string containing a networkstatus consensus, and the results of + * having parsed that consensus, add that consensus to the cache if it is not + * already present and not too old. Create new consensus diffs from or to + * that consensus as appropriate. + * + * Return 0 on success and -1 on failure. + */ +int +consdiffmgr_add_consensus(const char *consensus, + const networkstatus_t *as_parsed) +{ + if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) + return -1; // LCOV_EXCL_LINE + if (BUG(as_parsed->type != NS_TYPE_CONSENSUS)) + return -1; // LCOV_EXCL_LINE + + const consensus_flavor_t flavor = as_parsed->flavor; + const time_t valid_after = as_parsed->valid_after; + + if (valid_after < approx_time() - 3600 * consdiff_cfg.cache_max_age_hours) { + log_info(LD_DIRSERV, "We don't care about this consensus document; it's " + "too old."); + return -1; + } + + /* Do we already have this one? */ + consensus_cache_entry_t *entry = + cdm_cache_lookup_consensus(flavor, valid_after); + if (entry) { + log_info(LD_DIRSERV, "We already have a copy of that consensus"); + return -1; + } + + /* We don't have it. Add it to the cache. */ + { + size_t bodylen = strlen(consensus); + config_line_t *labels = NULL; + char formatted_time[ISO_TIME_LEN+1]; + format_iso_time_nospace(formatted_time, valid_after); + const char *flavname = networkstatus_get_flavor_name(flavor); + + cdm_labels_prepend_sha3(&labels, (const uint8_t *)consensus, bodylen); + config_line_prepend(&labels, LABEL_FLAVOR, flavname); + config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); + config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + entry = consensus_cache_add(cdm_cache_get(), + labels, + (const uint8_t *)consensus, + bodylen); + config_free_lines(labels); + } + + if (entry) { + consensus_cache_entry_mark_for_aggressive_release(entry); + consensus_cache_entry_decref(entry); + } + + cdm_cache_dirty = 1; + return entry ? 0 : -1; +} + +/** + * Helper: used to sort two smartlists of consensus_cache_entry_t by their + * LABEL_VALID_AFTER labels. + */ +static int +compare_by_valid_after_(const void **a, const void **b) +{ + const consensus_cache_entry_t *e1 = *a; + const consensus_cache_entry_t *e2 = *b; + /* We're in luck here: sorting UTC iso-encoded values lexically will work + * fine (until 9999). */ + return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER), + consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER)); +} + +/** + * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent + * entry. + */ +static consensus_cache_entry_t * +sort_and_find_most_recent(smartlist_t *lst) +{ + smartlist_sort(lst, compare_by_valid_after_); + if (smartlist_len(lst)) { + return smartlist_get(lst, smartlist_len(lst) - 1); + } else { + return NULL; + } +} + +/** + * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>, + * from the source consensus with the specified digest (which must be SHA3). + * + * If the diff is present, store it into *<b>entry_out</b> and return + * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or + * CONSDIFF_IN_PROGRESS. + */ +consdiff_status_t +consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + int digest_type, + const uint8_t *digest, + size_t digestlen) +{ + if (BUG(digest_type != DIGEST_SHA3_256) || + BUG(digestlen != DIGEST256_LEN)) { + return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE + } + + // Try to look up the entry in the hashtable. + cdm_diff_t search, *ent; + memset(&search, 0, sizeof(search)); + search.flavor = flavor; + memcpy(search.from_sha3, digest, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + + if (ent == NULL || + ent->cdm_diff_status == CDM_DIFF_ERROR) { + return CONSDIFF_NOT_FOUND; + } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) { + return CONSDIFF_IN_PROGRESS; + } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) { + return CONSDIFF_IN_PROGRESS; + } + + *entry_out = consensus_cache_entry_handle_get(ent->entry); + return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND; + +#if 0 + // XXXX Remove this. I'm keeping it around for now in case we need to + // XXXX debug issues in the hashtable. + char hex[HEX_DIGEST256_LEN+1]; + base16_encode(hex, sizeof(hex), (const char *)digest, digestlen); + const char *flavname = networkstatus_get_flavor_name(flavor); + + smartlist_t *matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FROM_SHA3_DIGEST, hex); + consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + + *entry_out = sort_and_find_most_recent(matches); + consdiff_status_t result = + (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND; + smartlist_free(matches); + + return result; +#endif +} + +/** + * Perform periodic cleanup tasks on the consensus diff cache. Return + * the number of objects marked for deletion. + */ +int +consdiffmgr_cleanup(void) +{ + smartlist_t *objects = smartlist_new(); + smartlist_t *consensuses = smartlist_new(); + smartlist_t *diffs = smartlist_new(); + int n_to_delete = 0; + + log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove"); + + // 1. Delete any consensus or diff or anything whose valid_after is too old. + const time_t valid_after_cutoff = + approx_time() - 3600 * consdiff_cfg.cache_max_age_hours; + + consensus_cache_find_all(objects, cdm_cache_get(), + NULL, NULL); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + const char *lv_valid_after = + consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER); + if (! lv_valid_after) { + log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label", + LABEL_VALID_AFTER); + continue; + } + time_t valid_after = 0; + if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) { + log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was " + "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after)); + continue; + } + if (valid_after < valid_after_cutoff) { + log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was " + "too old", LABEL_VALID_AFTER, lv_valid_after); + consensus_cache_entry_mark_for_removal(ent); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(ent); + + // 2. Delete all diffs that lead to a consensus whose valid-after is not the + // latest. + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + /* Determine the most recent consensus of this flavor */ + consensus_cache_find_all(consensuses, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname); + consensus_cache_entry_t *most_recent = + sort_and_find_most_recent(consensuses); + if (most_recent == NULL) + continue; + const char *most_recent_sha3 = + consensus_cache_entry_get_value(most_recent, LABEL_SHA3_DIGEST); + if (BUG(most_recent_sha3 == NULL)) + continue; // LCOV_EXCL_LINE + + /* consider all such-flavored diffs, and look to see if they match. */ + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *this_diff_target_sha3 = + consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST); + if (!this_diff_target_sha3) + continue; + if (strcmp(this_diff_target_sha3, most_recent_sha3)) { + consensus_cache_entry_mark_for_removal(diff); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(diff); + smartlist_clear(consensuses); + smartlist_clear(diffs); + } + + smartlist_free(objects); + smartlist_free(consensuses); + smartlist_free(diffs); + + // Actually remove files, if they're not used. + consensus_cache_delete_pending(cdm_cache_get()); + return n_to_delete; +} + +/** + * Initialize the consensus diff manager and its cache, and configure + * its parameters based on the latest torrc and networkstatus parameters. + */ +void +consdiffmgr_configure(const consdiff_cfg_t *cfg) +{ + memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg)); + + (void) cdm_cache_get(); +} + +/** + * Scan the consensus diff manager's cache for any grossly malformed entries, + * and mark them as deletable. Return 0 if no problems were found; 1 + * if problems were found and fixed. + */ +int +consdiffmgr_validate(void) +{ + /* Right now, we only check for entries that have bad sha3 values */ + int problems = 0; + + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cdm_cache_get(), + NULL, NULL); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) { + uint8_t sha3_expected[DIGEST256_LEN]; + uint8_t sha3_received[DIGEST256_LEN]; + int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST); + if (r == -1) { + /* digest isn't there; that's allowed */ + continue; + } else if (r == -2) { + /* digest is malformed; that's not allowed */ + problems = 1; + consensus_cache_entry_mark_for_removal(obj); + continue; + } + const uint8_t *body; + size_t bodylen; + consensus_cache_entry_incref(obj); + r = consensus_cache_entry_get_body(obj, &body, &bodylen); + if (r == 0) { + crypto_digest256((char *)sha3_received, (const char *)body, bodylen, + DIGEST_SHA3_256); + } + consensus_cache_entry_decref(obj); + if (r < 0) + continue; + + // Deconfuse coverity about the possibility of sha3_received being + // uninitialized + tor_assert(r <= 0); + + if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) { + problems = 1; + consensus_cache_entry_mark_for_removal(obj); + continue; + } + + } SMARTLIST_FOREACH_END(obj); + smartlist_free(objects); + return problems; +} + +/** + * Helper: build new diffs of <b>flavor</b> as needed + */ +static void +consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) +{ + smartlist_t *matches = NULL; + smartlist_t *diffs = NULL; + smartlist_t *compute_diffs_from = NULL; + strmap_t *have_diff_from = NULL; + + // look for the most recent consensus, and for all previous in-range + // consensuses. Do they all have diffs to it? + const char *flavname = networkstatus_get_flavor_name(flavor); + + // 1. find the most recent consensus, and the ones that we might want + // to diff to it. + matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); + if (!most_recent) { + log_info(LD_DIRSERV, "No 'most recent' %s consensus found; " + "not making diffs", flavname); + goto done; + } + tor_assert(smartlist_len(matches)); + smartlist_del(matches, smartlist_len(matches) - 1); + + const char *most_recent_valid_after = + consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER); + if (BUG(most_recent_valid_after == NULL)) + goto done; //LCOV_EXCL_LINE + uint8_t most_recent_sha3[DIGEST256_LEN]; + if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent, + LABEL_SHA3_DIGEST) < 0)) + goto done; //LCOV_EXCL_LINE + + // 2. Find all the relevant diffs _to_ this consensus. These are ones + // that we don't need to compute. + diffs = smartlist_new(); + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_VALID_AFTER, most_recent_valid_after); + consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname); + have_diff_from = strmap_new(); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *va = consensus_cache_entry_get_value(diff, + LABEL_FROM_VALID_AFTER); + if (BUG(va == NULL)) + continue; // LCOV_EXCL_LINE + strmap_set(have_diff_from, va, diff); + } SMARTLIST_FOREACH_END(diff); + + // 3. See which consensuses in 'matches' don't have diffs yet. + smartlist_reverse(matches); // from newest to oldest. + compute_diffs_from = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) { + const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER); + if (BUG(va == NULL)) + continue; // LCOV_EXCL_LINE + if (strmap_get(have_diff_from, va) != NULL) + continue; /* we already have this one. */ + smartlist_add(compute_diffs_from, ent); + } SMARTLIST_FOREACH_END(ent); + + log_info(LD_DIRSERV, + "The most recent %s consensus is valid-after %s. We have diffs to " + "this consensus for %d/%d older %s consensuses. Generating diffs " + "for the other %d.", + flavname, + most_recent_valid_after, + smartlist_len(matches) - smartlist_len(compute_diffs_from), + smartlist_len(matches), + flavname, + smartlist_len(compute_diffs_from)); + + // 4. Update the hashtable; remove entries in this flavor to other + // target consensuses. + cdm_diff_ht_purge(flavor, most_recent_sha3); + + // 5. Actually launch the requests. + SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) { + if (BUG(c == most_recent)) + continue; // LCOV_EXCL_LINE + + uint8_t this_sha3[DIGEST256_LEN]; + if (BUG(cdm_entry_get_sha3_value(this_sha3, c, LABEL_SHA3_DIGEST)<0)) + continue; // LCOV_EXCL_LINE + if (cdm_diff_ht_check_and_note_pending(flavor, + this_sha3, most_recent_sha3)) { + // This is already pending, or we encountered an error. + continue; + } + consensus_diff_queue_diff_work(c, most_recent); + } SMARTLIST_FOREACH_END(c); + + done: + smartlist_free(matches); + smartlist_free(diffs); + smartlist_free(compute_diffs_from); + strmap_free(have_diff_from, NULL); +} + +/** + * Scan the cache for diffs, and add them to the hashtable. + */ +static void +consdiffmgr_diffs_load(void) +{ + smartlist_t *diffs = smartlist_new(); + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *lv_flavor = + consensus_cache_entry_get_value(diff, LABEL_FLAVOR); + if (!lv_flavor) + continue; + int flavor = networkstatus_parse_flavor_name(lv_flavor); + if (flavor < 0) + continue; + uint8_t from_sha3[DIGEST256_LEN]; + uint8_t to_sha3[DIGEST256_LEN]; + if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0) + continue; + if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0) + continue; + + cdm_diff_ht_set_status(flavor, from_sha3, to_sha3, + CDM_DIFF_PRESENT, + consensus_cache_entry_handle_new(diff)); + } SMARTLIST_FOREACH_END(diff); + smartlist_free(diffs); +} + +/** + * Build new diffs as needed. + */ +void +consdiffmgr_rescan(void) +{ + if (cdm_cache_dirty == 0) + return; + + // Clean up here to make room for new diffs, and to ensure that older + // consensuses do not have any entries. + consdiffmgr_cleanup(); + + if (cdm_cache_loaded == 0) { + consdiffmgr_diffs_load(); + cdm_cache_loaded = 1; + } + + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + consdiffmgr_rescan_flavor_((consensus_flavor_t) flav); + } + + cdm_cache_dirty = 0; +} + +/** + * Set consensus cache flags on the objects in this consdiffmgr. + */ +static void +consdiffmgr_set_cache_flags(void) +{ + /* Right now, we just mark the consensus objects for aggressive release, + * so that they get mmapped for as little time as possible. */ + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE, + DOCTYPE_CONSENSUS); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + consensus_cache_entry_mark_for_aggressive_release(ent); + } SMARTLIST_FOREACH_END(ent); + smartlist_free(objects); +} + +/** + * Called before shutdown: drop all storage held by the consdiffmgr.c module. + */ +void +consdiffmgr_free_all(void) +{ + cdm_diff_t **diff, **next; + for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) { + cdm_diff_t *this = *diff; + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + } + consensus_cache_free(cons_diff_cache); + cons_diff_cache = NULL; +} + +/* ===== + Thread workers + =====*/ + +/** + * An object passed to a worker thread that will try to produce a consensus + * diff. + */ +typedef struct consensus_diff_worker_job_t { + /** + * Input: The consensus to compute the diff from. Holds a reference to the + * cache entry, which must not be released until the job is passed back to + * the main thread. The body must be mapped into memory in the main thread. + */ + consensus_cache_entry_t *diff_from; + /** + * Input: The consensus to compute the diff to. Holds a reference to the + * cache entry, which must not be released until the job is passed back to + * the main thread. The body must be mapped into memory in the main thread. + */ + consensus_cache_entry_t *diff_to; + + /** + * Output: Labels to store in the cache associated with this diff. + */ + config_line_t *labels_out; + /** + * Output: Body of the diff + */ + uint8_t *body_out; + /** + * Output: length of body_out + */ + size_t bodylen_out; +} consensus_diff_worker_job_t; + +/** + * Worker function. This function runs inside a worker thread and receives + * a consensus_diff_worker_job_t as its input. + */ +static workqueue_reply_t +consensus_diff_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + consensus_diff_worker_job_t *job = work_; + const uint8_t *diff_from, *diff_to; + size_t len_from, len_to; + int r; + /* We need to have the body already mapped into RAM here. + */ + r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from); + if (BUG(r < 0)) + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to); + if (BUG(r < 0)) + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + + const char *lv_to_valid_after = + consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER); + const char *lv_from_valid_after = + consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER); + const char *lv_from_digest = + consensus_cache_entry_get_value(job->diff_from, LABEL_SHA3_DIGEST); + const char *lv_from_flavor = + consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR); + const char *lv_to_flavor = + consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR); + const char *lv_to_digest = + consensus_cache_entry_get_value(job->diff_to, LABEL_SHA3_DIGEST); + + /* All these values are mandatory on the input */ + if (BUG(!lv_to_valid_after) || + BUG(!lv_from_valid_after) || + BUG(!lv_from_digest) || + BUG(!lv_from_flavor) || + BUG(!lv_to_flavor)) { + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + /* The flavors need to match */ + if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) { + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + + char *consensus_diff; + { + // XXXX the input might not be nul-terminated. And also we wanted to + // XXXX support compression later I guess. So, we need to copy here. + char *diff_from_nt, *diff_to_nt; + diff_from_nt = tor_memdup_nulterm(diff_from, len_from); + diff_to_nt = tor_memdup_nulterm(diff_to, len_to); + + // XXXX ugh; this is going to calculate the SHA3 of both its + // XXXX inputs again, even though we already have that. Maybe it's time + // XXXX to change the API here? + consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); + tor_free(diff_from_nt); + tor_free(diff_to_nt); + } + if (!consensus_diff) { + /* Couldn't generate consensus; we'll leave the reply blank. */ + return WQ_RPL_REPLY; + } + + /* Send the reply */ + job->body_out = (uint8_t *) consensus_diff; + job->bodylen_out = strlen(consensus_diff); + + cdm_labels_prepend_sha3(&job->labels_out, job->body_out, job->bodylen_out); + config_line_prepend(&job->labels_out, LABEL_FROM_VALID_AFTER, + lv_from_valid_after); + config_line_prepend(&job->labels_out, LABEL_VALID_AFTER, lv_to_valid_after); + config_line_prepend(&job->labels_out, LABEL_FLAVOR, lv_from_flavor); + config_line_prepend(&job->labels_out, LABEL_FROM_SHA3_DIGEST, + lv_from_digest); + config_line_prepend(&job->labels_out, LABEL_TARGET_SHA3_DIGEST, + lv_to_digest); + config_line_prepend(&job->labels_out, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + return WQ_RPL_REPLY; +} + +/** + * Helper: release all storage held in <b>job</b>. + */ +static void +consensus_diff_worker_job_free(consensus_diff_worker_job_t *job) +{ + if (!job) + return; + tor_free(job->body_out); + config_free_lines(job->labels_out); + consensus_cache_entry_decref(job->diff_from); + consensus_cache_entry_decref(job->diff_to); + tor_free(job); +} + +/** + * Worker function: This function runs in the main thread, and receives + * a consensus_diff_worker_job_t that the worker thread has already + * processed. + */ +static void +consensus_diff_worker_replyfn(void *work_) +{ + tor_assert(in_main_thread()); + tor_assert(work_); + + consensus_diff_worker_job_t *job = work_; + + const char *lv_from_digest = + consensus_cache_entry_get_value(job->diff_from, LABEL_SHA3_DIGEST); + const char *lv_to_digest = + consensus_cache_entry_get_value(job->diff_to, LABEL_SHA3_DIGEST); + const char *lv_flavor = + consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR); + if (BUG(lv_from_digest == NULL)) + lv_from_digest = "???"; // LCOV_EXCL_LINE + if (BUG(lv_to_digest == NULL)) + lv_to_digest = "???"; // LCOV_EXCL_LINE + + uint8_t from_sha3[DIGEST256_LEN]; + uint8_t to_sha3[DIGEST256_LEN]; + int flav = -1; + int cache = 1; + if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from, + LABEL_SHA3_DIGEST) < 0)) + cache = 0; + if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to, + LABEL_SHA3_DIGEST) < 0)) + cache = 0; + if (BUG(lv_flavor == NULL)) { + cache = 0; + } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) { + cache = 0; + } + + int status; + consensus_cache_entry_handle_t *handle = NULL; + if (job->body_out && job->bodylen_out && job->labels_out) { + /* Success! Store the results */ + log_info(LD_DIRSERV, "Adding consensus diff from %s to %s", + lv_from_digest, lv_to_digest); + consensus_cache_entry_t *ent = + consensus_cache_add(cdm_cache_get(), job->labels_out, + job->body_out, + job->bodylen_out); + status = CDM_DIFF_PRESENT; + handle = consensus_cache_entry_handle_new(ent); + consensus_cache_entry_decref(ent); + } else { + /* Failure! Nothing to do but complain */ + log_warn(LD_DIRSERV, + "Worker was unable to compute consensus diff " + "from %s to %s", lv_from_digest, lv_to_digest); + /* Cache this error so we don't try to compute this one again. */ + status = CDM_DIFF_ERROR; + } + + if (cache) + cdm_diff_ht_set_status(flav, from_sha3, to_sha3, status, handle); + else + consensus_cache_entry_handle_free(handle); + + consensus_diff_worker_job_free(job); +} + +/** + * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b> + * in a worker thread. + */ +static int +consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, + consensus_cache_entry_t *diff_to) +{ + tor_assert(in_main_thread()); + + consensus_cache_entry_incref(diff_from); + consensus_cache_entry_incref(diff_to); + + consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->diff_from = diff_from; + job->diff_to = diff_to; + + /* Make sure body is mapped. */ + const uint8_t *body; + size_t bodylen; + int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen); + int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen); + if (r1 < 0 || r2 < 0) + goto err; + + workqueue_entry_t *work; + work = cpuworker_queue_work(consensus_diff_worker_threadfn, + consensus_diff_worker_replyfn, + job); + if (!work) + goto err; + + return 0; + err: + consensus_diff_worker_job_free(job); // includes decrefs. + return -1; +} + diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h new file mode 100644 index 0000000000..6932b2fba3 --- /dev/null +++ b/src/or/consdiffmgr.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONSDIFFMGR_H +#define TOR_CONSDIFFMGR_H + +/** + * Possible outcomes from trying to look up a given consensus diff. + */ +typedef enum consdiff_status_t { + CONSDIFF_AVAILABLE, + CONSDIFF_NOT_FOUND, + CONSDIFF_IN_PROGRESS, +} consdiff_status_t; + +typedef struct consdiff_cfg_t { + int32_t cache_max_age_hours; + int32_t cache_max_num; +} consdiff_cfg_t; + +struct consensus_cache_entry_t; // from conscache.h + +int consdiffmgr_add_consensus(const char *consensus, + const networkstatus_t *as_parsed); + +consdiff_status_t consdiffmgr_find_diff_from( + struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + int digest_type, + const uint8_t *digest, + size_t digestlen); +void consdiffmgr_rescan(void); +int consdiffmgr_cleanup(void); +void consdiffmgr_configure(const consdiff_cfg_t *cfg); +void consdiffmgr_free_all(void); +int consdiffmgr_validate(void); + +#ifdef CONSDIFFMGR_PRIVATE +STATIC consensus_cache_t *cdm_cache_get(void); +STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( + consensus_flavor_t flavor, time_t valid_after); +STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, + consensus_cache_entry_t *ent, + const char *label); +#endif + +#endif + diff --git a/src/or/control.c b/src/or/control.c index 30067293dc..56f1329029 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1873,7 +1873,7 @@ getinfo_helper_listeners(control_connection_t *control_conn, /** Implementation helper for GETINFO: knows the answers for questions about * directory information. */ -static int +STATIC int getinfo_helper_dir(control_connection_t *control_conn, const char *question, char **answer, const char **errmsg) @@ -2064,7 +2064,7 @@ getinfo_helper_dir(control_connection_t *control_conn, char d[DIGEST_LEN]; signed_descriptor_t *sd = NULL; if (base16_decode(d, sizeof(d), question, strlen(question)) - != sizeof(d)) { + == sizeof(d)) { /* XXXX this test should move into extrainfo_get_by_descriptor_digest, * but I don't want to risk affecting other parts of the code, * especially since the rules for using our own extrainfo (including diff --git a/src/or/control.h b/src/or/control.h index a786dfe1af..41a194bfcb 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -290,6 +290,10 @@ STATIC int getinfo_helper_downloads( control_connection_t *control_conn, const char *question, char **answer, const char **errmsg); +STATIC int getinfo_helper_dir( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); #endif diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 790c147d7b..af79fafaa6 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -479,6 +479,20 @@ queue_pending_tasks(void) } } +/** DOCDOC */ +MOCK_IMPL(workqueue_entry_t *, +cpuworker_queue_work,(workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg)) +{ + tor_assert(threadpool); + + return threadpool_queue_work(threadpool, + fn, + reply_fn, + arg); +} + /** Try to tell a cpuworker to perform the public key operations necessary to * respond to <b>onionskin</b> for the circuit <b>circ</b>. * diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h index fd426e21f6..aedf2fae32 100644 --- a/src/or/cpuworker.h +++ b/src/or/cpuworker.h @@ -14,6 +14,12 @@ void cpu_init(void); void cpuworkers_rotate_keyinfo(void); +struct workqueue_entry_s; +enum workqueue_reply_t; +MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, ( + enum workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg)); struct create_cell_t; int assign_onionskin_to_cpuworker(or_circuit_t *circ, diff --git a/src/or/directory.c b/src/or/directory.c index 872cce2c9e..751d46910e 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2235,15 +2235,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } /* Try declared compression first if we can. */ if (compression == GZIP_METHOD || compression == ZLIB_METHOD) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, compression, + !allow_partial, LOG_PROTOCOL_WARN); /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ if (!new_body && (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && compression != guessed) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, guessed, + !allow_partial, LOG_PROTOCOL_WARN); /* If we're pretty sure that we have a compressed directory, and * we didn't manage to uncompress it, then warn and bail. */ if (!plausible && !new_body) { @@ -2984,7 +2984,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) /** Return the compression level we should use for sending a compressed * response of size <b>n_bytes</b>. */ -STATIC zlib_compression_level_t +STATIC compression_level_t choose_compression_level(ssize_t n_bytes) { if (! have_been_under_memory_pressure()) { @@ -3317,7 +3317,8 @@ handle_get_current_consensus(dir_connection_t *conn, write_http_response_header(conn, -1, compressed, smartlist_len(conn->spool) == 1 ? lifetime : 0); if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + conn->compress_state = tor_compress_new(0, ZLIB_METHOD, + HIGH_COMPRESSION); /* Prime the connection with some data. */ const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3415,11 +3416,11 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(estimated_len)); + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf_zlib(c, strlen(c), conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress(c, strlen(c), conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf(c, strlen(c), TO_CONN(conn))); @@ -3474,7 +3475,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3567,7 +3568,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) } write_http_response_header(conn, -1, compressed, cache_lifetime); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -3658,13 +3659,14 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, compressed?-1:len, compressed, 60*60); if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(len)); + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress( + c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf(c->cache_info.signed_descriptor_body, diff --git a/src/or/directory.h b/src/or/directory.h index 291ced4191..e4fbbe6891 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -185,7 +185,7 @@ STATIC int handle_post_hs_descriptor(const char *url, const char *body); STATIC char* authdir_type_to_string(dirinfo_type_t auth); STATIC const char * dir_conn_purpose_to_string(int purpose); STATIC int should_use_directory_guards(const or_options_t *options); -STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC compression_level_t choose_compression_level(ssize_t n_bytes); STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, const or_options_t *options); STATIC void find_dl_min_and_max_delay(download_status_t *dls, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 70b0b22f25..e76fd932ca 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1176,8 +1176,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, + ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; @@ -3497,8 +3497,8 @@ spooled_resource_flush_some(spooled_resource_t *spooled, /* Absent objects count as "done". */ return SRFS_DONE; } - if (conn->zlib_state) { - connection_write_to_buf_zlib((const char*)body, bodylen, conn, 0); + if (conn->compress_state) { + connection_write_to_buf_compress((const char*)body, bodylen, conn, 0); } else { connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn)); } @@ -3523,9 +3523,10 @@ spooled_resource_flush_some(spooled_resource_t *spooled, if (BUG(remaining < 0)) return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); - if (conn->zlib_state) { - connection_write_to_buf_zlib(cached->dir_z + spooled->cached_dir_offset, - bytes, conn, 0); + if (conn->compress_state) { + connection_write_to_buf_compress( + cached->dir_z + spooled->cached_dir_offset, + bytes, conn, 0); } else { connection_write_to_buf(cached->dir_z + spooled->cached_dir_offset, bytes, TO_CONN(conn)); @@ -3788,12 +3789,12 @@ connection_dirserv_flushed_some(dir_connection_t *conn) /* If we get here, we're done. */ smartlist_free(conn->spool); conn->spool = NULL; - if (conn->zlib_state) { - /* Flush the zlib state: there could be more bytes pending in there, and - * we don't want to omit bytes. */ - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; + if (conn->compress_state) { + /* Flush the compression state: there could be more bytes pending in there, + * and we don't want to omit bytes. */ + connection_write_to_buf_compress("", 0, conn, 1); + tor_compress_free(conn->compress_state); + conn->compress_state = NULL; } return 0; } diff --git a/src/or/hs_common.c b/src/or/hs_common.c index 4af3081502..42508126f8 100644 --- a/src/or/hs_common.c +++ b/src/or/hs_common.c @@ -9,6 +9,8 @@ * protocol. **/ +#define HS_COMMON_PRIVATE + #include "or.h" #include "config.h" @@ -50,6 +52,46 @@ hs_check_service_private_dir(const char *username, const char *path, return 0; } +/** Get the default HS time period length in minutes from the consensus. */ +STATIC uint64_t +get_time_period_length(void) +{ + int32_t time_period_length = networkstatus_get_param(NULL, "hsdir-interval", + HS_TIME_PERIOD_LENGTH_DEFAULT, + HS_TIME_PERIOD_LENGTH_MIN, + HS_TIME_PERIOD_LENGTH_MAX); + /* Make sure it's a positive value. */ + tor_assert(time_period_length >= 0); + /* uint64_t will always be able to contain a int32_t */ + return (uint64_t) time_period_length; +} + +/** Get the HS time period number at time <b>now</b> */ +STATIC uint64_t +get_time_period_num(time_t now) +{ + uint64_t time_period_num; + uint64_t time_period_length = get_time_period_length(); + uint64_t minutes_since_epoch = now / 60; + + /* Now subtract half a day to fit the prop224 time period schedule (see + * section [TIME-PERIODS]). */ + tor_assert(minutes_since_epoch > HS_TIME_PERIOD_ROTATION_OFFSET); + minutes_since_epoch -= HS_TIME_PERIOD_ROTATION_OFFSET; + + /* Calculate the time period */ + time_period_num = minutes_since_epoch / time_period_length; + return time_period_num; +} + +/** Get the number of the _upcoming_ HS time period, given that the current + * time is <b>now</b>. */ +uint64_t +hs_get_next_time_period_num(time_t now) +{ + return get_time_period_num(now) + 1; +} + /* Create a new rend_data_t for a specific given <b>version</b>. * Return a pointer to the newly allocated data structure. */ static rend_data_t * diff --git a/src/or/hs_common.h b/src/or/hs_common.h index d7d4228da2..a8fded652a 100644 --- a/src/or/hs_common.h +++ b/src/or/hs_common.h @@ -40,6 +40,15 @@ /* String prefix for the signature of ESTABLISH_INTRO */ #define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1" +/* The default HS time period length */ +#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */ +/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */ +/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */ +/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */ + int hs_check_service_private_dir(const char *username, const char *path, unsigned int dir_group_readable, unsigned int create); @@ -60,5 +69,18 @@ const char *rend_data_get_desc_id(const rend_data_t *rend_data, const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out); +uint64_t hs_get_next_time_period_num(time_t now); + +#ifdef HS_COMMON_PRIVATE + +#ifdef TOR_UNIT_TESTS + +STATIC uint64_t get_time_period_length(void); +STATIC uint64_t get_time_period_num(time_t now); + +#endif /* TOR_UNIT_TESTS */ + +#endif /* HS_COMMON_PRIVATE */ + #endif /* TOR_HS_COMMON_H */ diff --git a/src/or/include.am b/src/or/include.am index 1841bbfe9d..4c24dd23b3 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -38,6 +38,7 @@ LIBTOR_A_SOURCES = \ src/or/connection_or.c \ src/or/conscache.c \ src/or/consdiff.c \ + src/or/consdiffmgr.c \ src/or/control.c \ src/or/cpuworker.c \ src/or/dircollate.c \ @@ -119,8 +120,10 @@ src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libev src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ + src/trace/libor-trace.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -132,7 +135,8 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ endif ORHEADERS = \ @@ -156,6 +160,7 @@ ORHEADERS = \ src/or/connection_or.h \ src/or/conscache.h \ src/or/consdiff.h \ + src/or/consdiffmgr.h \ src/or/control.h \ src/or/cpuworker.h \ src/or/dircollate.h \ diff --git a/src/or/main.c b/src/or/main.c index 4505879adc..5fec7e4a5d 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -58,6 +58,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compress.h" #include "config.h" #include "confparse.h" #include "connection.h" @@ -2998,11 +2999,16 @@ tor_init(int argc, char *argv[]) const char *version = get_version(); log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, " - "OpenSSL %s and Zlib %s.", version, + "OpenSSL %s, Zlib %s, Liblzma %s, and Libzstd %s.", version, get_uname(), tor_libevent_get_version_str(), crypto_openssl_get_version_str(), - tor_zlib_get_version_str()); + tor_compress_supports_method(ZLIB_METHOD) ? + tor_compress_version_str(ZLIB_METHOD) : "N/A", + tor_compress_supports_method(LZMA_METHOD) ? + tor_compress_version_str(LZMA_METHOD) : "N/A", + tor_compress_supports_method(ZSTD_METHOD) ? + tor_compress_version_str(ZSTD_METHOD) : "N/A"); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " @@ -3610,6 +3616,7 @@ tor_main(int argc, char *argv[]) update_approx_time(time(NULL)); tor_threads_init(); + tor_compress_init(); init_logging(0); monotime_init(); #ifdef USE_DMALLOC diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 9a65fcf014..294fc0df6d 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -159,7 +159,7 @@ onion_skin_TAP_server_handshake( * big. That should be impossible. */ log_info(LD_GENERAL, "crypto_dh_get_public failed."); goto err; - /* LCOV_EXCP_STOP */ + /* LCOV_EXCL_STOP */ } key_material_len = DIGEST_LEN+key_out_len; diff --git a/src/or/or.h b/src/or/or.h index 5a42aada50..a09533f970 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -71,7 +71,7 @@ #include "tortls.h" #include "torlog.h" #include "container.h" -#include "torgzip.h" +#include "compress.h" #include "address.h" #include "compat_libevent.h" #include "ht.h" @@ -1774,8 +1774,8 @@ typedef struct dir_connection_t { /** List of spooled_resource_t for objects that we're spooling. We use * it from back to front. */ smartlist_t *spool; - /** The zlib object doing on-the-fly compression for spooled data. */ - tor_zlib_state_t *zlib_state; + /** The compression object doing on-the-fly compression for spooled data. */ + tor_compress_state_t *compress_state; /** What rendezvous service are we querying for? */ rend_data_t *rend_data; diff --git a/src/or/relay.c b/src/or/relay.c index 5139036327..1842012ed7 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -54,6 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -225,7 +226,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) { - log_warn(LD_BUG,"relay crypt failed. Dropping connection."); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "relay crypt failed. Dropping connection."); return -END_CIRC_REASON_INTERNAL; } @@ -2453,7 +2455,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); - alloc += tor_zlib_get_total_allocation(); + alloc += tor_compress_get_total_allocation(); const size_t rend_cache_total = rend_cache_get_total_allocation(); alloc += rend_cache_total; if (alloc >= get_options()->MaxMemInQueues_low_threshold) { diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 82d8e4c48e..5de153caa4 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1054,20 +1054,13 @@ rend_log_intro_limit(const rend_service_t *service, int min_severity) } time_t intro_period_elapsed = time(NULL) - service->intro_period_started; tor_assert_nonfatal(intro_period_elapsed >= 0); - /* We delayed resuming circuits longer than expected */ - int exceeded_elapsed = (intro_period_elapsed > INTRO_CIRC_RETRY_PERIOD + - INTRO_CIRC_RETRY_PERIOD_SLOP); - if (exceeded_elapsed) { - severity = LOG_WARN; - } log_fn(severity, LD_REND, "Hidden service %s %s %d intro points in the last " - "%d seconds%s. Intro circuit launches are limited to %d per %d " + "%d seconds. Intro circuit launches are limited to %d per %d " "seconds.", service->service_id, exceeded_limit ? "exceeded launch limit with" : "launched", service->n_intro_circuits_launched, (int)intro_period_elapsed, - exceeded_elapsed ? " (delayed)" : "", rend_max_intro_circs_per_period(service->n_intro_points_wanted), INTRO_CIRC_RETRY_PERIOD); rend_service_dump_stats(severity); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 060862f25c..8927786fe8 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -3048,8 +3048,8 @@ router_get_by_extrainfo_digest,(const char *digest)) /** Return the signed descriptor for the extrainfo_t in our routerlist whose * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info * document is known. */ -signed_descriptor_t * -extrainfo_get_by_descriptor_digest(const char *digest) +MOCK_IMPL(signed_descriptor_t *, +extrainfo_get_by_descriptor_digest,(const char *digest)) { extrainfo_t *ei; tor_assert(digest); diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 5376369e8d..5c1f76c8e7 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -92,7 +92,8 @@ routerinfo_t *router_get_mutable_by_digest(const char *digest); signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest, (const char *digest)); -signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); +MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest, + (const char *digest)); const char *signed_descriptor_get_body(const signed_descriptor_t *desc); const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc); routerlist_t *router_get_routerlist(void); diff --git a/src/test/bench.c b/src/test/bench.c index 5da9168cdc..a44dc94a61 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -674,6 +674,7 @@ main(int argc, const char **argv) or_options_t *options; tor_threads_init(); + tor_compress_init(); if (argc == 4 && !strcmp(argv[1], "diff")) { init_logging(1); diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index f540176204..7aee92df63 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -96,6 +96,7 @@ static void global_init(void) { tor_threads_init(); + tor_compress_init(); { struct sipkey sipkey = { 1337, 7331 }; siphash_set_global_key(&sipkey); diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 1b608c6885..500377f6d7 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -18,7 +18,9 @@ FUZZING_LIBS = \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ \ + @TOR_ZSTD_LIBS@ oss-fuzz-prereqs: \ src/or/libtor-testing.a \ diff --git a/src/test/include.am b/src/test/include.am index c92eab13c9..230a6c8bad 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -89,6 +89,7 @@ src_test_test_SOURCES = \ src/test/test_connection.c \ src/test/test_conscache.c \ src/test/test_consdiff.c \ + src/test/test_consdiffmgr.c \ src/test/test_containers.c \ src/test/test_controller.c \ src/test/test_controller_events.c \ @@ -179,7 +180,8 @@ src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@ src_test_test_switch_id_LDADD = \ src/common/libor-testing.a \ src/common/libor-ctime-testing.a \ - @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -191,9 +193,10 @@ src_test_test_LDADD = src/or/libtor-testing.a \ src/common/libor-ctime-testing.a \ src/common/libor-event-testing.a \ src/trunnel/libor-trunnel-testing.a \ + src/trace/libor-trace.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) @@ -214,9 +217,10 @@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ + src/trace/libor-trace.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -225,8 +229,10 @@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \ src/common/libor-ctime-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a \ + src/trace/libor-trace.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS) @@ -236,7 +242,8 @@ src_test_test_timers_LDADD = \ src/common/libor-event-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS) noinst_HEADERS+= \ @@ -260,8 +267,10 @@ src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ + src/trace/libor-trace.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_ntor_cl_AM_CPPFLAGS = \ -I"$(top_srcdir)/src/or" @@ -280,6 +289,7 @@ noinst_PROGRAMS += src/test/test-bt-cl src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ src/common/libor-ctime-testing.a \ + src/trace/libor-trace.a \ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) diff --git a/src/test/test.c b/src/test/test.c index 77ea44970c..4d2cf1536b 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -44,13 +44,13 @@ double fabs(double x); #include "buffers.h" #include "circuitlist.h" #include "circuitstats.h" +#include "compress.h" #include "config.h" #include "connection_edge.h" #include "geoip.h" #include "rendcommon.h" #include "rendcache.h" #include "test.h" -#include "torgzip.h" #include "main.h" #include "memarea.h" #include "onion.h" @@ -1197,6 +1197,7 @@ struct testgroup_t testgroups[] = { { "connection/", connection_tests }, { "conscache/", conscache_tests }, { "consdiff/", consdiff_tests }, + { "consdiffmgr/", consdiffmgr_tests }, { "container/", container_tests }, { "control/", controller_tests }, { "control/event/", controller_event_tests }, diff --git a/src/test/test.h b/src/test/test.h index 252a239afc..3d7d05e771 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -192,6 +192,7 @@ extern struct testcase_t config_tests[]; extern struct testcase_t connection_tests[]; extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; +extern struct testcase_t consdiffmgr_tests[]; extern struct testcase_t container_tests[]; extern struct testcase_t controller_tests[]; extern struct testcase_t controller_event_tests[]; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index d14165c45c..43582d1b7a 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -584,22 +584,26 @@ test_buffers_zlib_impl(int finalize_with_nil) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_zlib_state_t *zlib_state = NULL; + tor_compress_state_t *compress_state = NULL; size_t out_len, in_len; int done; buf = buf_new_with_capacity(128); /* will round up */ - zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + compress_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); msg = tor_malloc(512); crypto_rand(msg, 512); - tt_int_op(write_to_buf_zlib(buf, zlib_state, msg, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+128, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+256, 256, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg+128, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg+256, 256, 0), OP_EQ, 0); done = !finalize_with_nil; - tt_int_op(write_to_buf_zlib(buf, zlib_state, "all done", 9, done), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + "all done", 9, done), OP_EQ, 0); if (finalize_with_nil) { - tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); } in_len = buf_datalen(buf); @@ -607,10 +611,10 @@ test_buffers_zlib_impl(int finalize_with_nil) tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0); - tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len, - contents, in_len, - ZLIB_METHOD, 1, - LOG_WARN)); + tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len, + contents, in_len, + ZLIB_METHOD, 1, + LOG_WARN)); tt_int_op(out_len, OP_GE, 128); tt_mem_op(msg, OP_EQ, expanded, 128); @@ -621,7 +625,7 @@ test_buffers_zlib_impl(int finalize_with_nil) done: buf_free(buf); - tor_zlib_free(zlib_state); + tor_compress_free(compress_state); tor_free(contents); tor_free(expanded); tor_free(msg); @@ -647,7 +651,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_zlib_state_t *zlib_state = NULL; + tor_compress_state_t *compress_state = NULL; size_t out_len, in_len; size_t sz, headerjunk; (void) arg; @@ -666,8 +670,8 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(buf->head->datalen, OP_EQ, headerjunk); tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk); /* Write an empty string, with finalization on. */ - zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); - tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0); + compress_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); in_len = buf_datalen(buf); contents = tor_malloc(in_len); @@ -676,17 +680,18 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(in_len, OP_GT, headerjunk); - tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len, - contents + headerjunk, in_len - headerjunk, - ZLIB_METHOD, 1, - LOG_WARN)); + tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len, + contents + headerjunk, + in_len - headerjunk, + ZLIB_METHOD, 1, + LOG_WARN)); tt_int_op(out_len, OP_EQ, 0); tt_assert(expanded); done: buf_free(buf); - tor_zlib_free(zlib_state); + tor_compress_free(compress_state); tor_free(contents); tor_free(expanded); tor_free(msg); diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c index 520aaf3c59..344ab27921 100644 --- a/src/test/test_circuitlist.c +++ b/src/test/test_circuitlist.c @@ -388,6 +388,7 @@ test_hs_circuitmap_isolation(void *arg) const uint8_t tok1[REND_TOKEN_LEN] = "bet i got some of th"; circ1 = or_circuit_new(0, NULL); + tt_assert(circ1); circ1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING; /* check that circuitmap is empty right? */ @@ -408,10 +409,13 @@ test_hs_circuitmap_isolation(void *arg) const uint8_t tok2[REND_TOKEN_LEN] = "you dont know anythi"; circ2 = origin_circuit_new(); + tt_assert(circ2); circ2->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO; circ3 = or_circuit_new(0, NULL); + tt_assert(circ3); circ3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT; circ4 = origin_circuit_new(); + tt_assert(circ4); circ4->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO; /* Register circ2 with tok2 as service-side intro v2 circ */ @@ -437,14 +441,10 @@ test_hs_circuitmap_isolation(void *arg) } done: - if (circ1) - circuit_free(TO_CIRCUIT(circ1)); - if (circ2) - circuit_free(TO_CIRCUIT(circ2)); - if (circ3) - circuit_free(TO_CIRCUIT(circ3)); - if (circ4) - circuit_free(TO_CIRCUIT(circ4)); + circuit_free(TO_CIRCUIT(circ1)); + circuit_free(TO_CIRCUIT(circ2)); + circuit_free(TO_CIRCUIT(circ3)); + circuit_free(TO_CIRCUIT(circ4)); } struct testcase_t circuitlist_tests[] = { diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c index 12184f0cc6..c316411a79 100644 --- a/src/test/test_conscache.c +++ b/src/test/test_conscache.c @@ -200,8 +200,7 @@ test_conscache_cleanup(void *arg) tt_assert(e_tmp); tt_assert(consensus_cache_entry_is_mapped(e_tmp)); e_tmp = consensus_cache_find_first(cache, "index", "7"); - tt_assert(e_tmp); - tt_assert(consensus_cache_entry_is_mapped(e_tmp)); + tt_assert(e_tmp == NULL); // not found because pending deletion. /* Delete the pending-deletion items. */ consensus_cache_delete_pending(cache); @@ -210,12 +209,12 @@ test_conscache_cleanup(void *arg) consensus_cache_find_all(entries, cache, NULL, NULL); int n = smartlist_len(entries); smartlist_free(entries); - tt_int_op(n, OP_EQ, 20 - 1); /* 1 entry was deleted */ + tt_int_op(n, OP_EQ, 20 - 2); /* 1 entry was deleted; 1 is not-found. */ } e_tmp = consensus_cache_find_first(cache, "index", "7"); // refcnt == 1... tt_assert(e_tmp == NULL); // so deleted. e_tmp = consensus_cache_find_first(cache, "index", "14"); // refcnt == 2 - tt_assert(e_tmp); // so, not deleted. + tt_assert(e_tmp == NULL); // not deleted; but not found. /* Now do lazy unmapping. */ // should do nothing. diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c index 1b4e2ad3c1..2f826b5ef7 100644 --- a/src/test/test_consdiff.c +++ b/src/test/test_consdiff.c @@ -622,6 +622,19 @@ test_consdiff_gen_ed_diff(void *arg) tt_str_eq_line(".", smartlist_get(diff, 5)); tt_str_eq_line("2d", smartlist_get(diff, 6)); + smartlist_free(diff); + + smartlist_clear(cons1); + smartlist_clear(cons2); + consensus_split_lines(cons1, "B\n", area); + consensus_split_lines(cons2, "A\nB\n", area); + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(3, OP_EQ, smartlist_len(diff)); + tt_str_eq_line("0a", smartlist_get(diff, 0)); + tt_str_eq_line("A", smartlist_get(diff, 1)); + tt_str_eq_line(".", smartlist_get(diff, 2)); + /* TODO: small real use-cases, i.e. consensuses. */ done: @@ -687,6 +700,15 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_clear(diff); + /* Unexpected range for add command. */ + smartlist_add_linecpy(diff, area, "1,2a"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("add lines after a range"); + + smartlist_clear(diff); + /* Script is not in reverse order. */ smartlist_add_linecpy(diff, area, "1d"); smartlist_add_linecpy(diff, area, "3d"); @@ -746,6 +768,55 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_clear(diff); + /* Ranges must be numeric only and cannot contain spaces. */ + smartlist_add_linecpy(diff, area, "0, 4d"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an ed command was missing a range " + "end line number."); + + smartlist_clear(diff); + + /* '+' is not a number. */ + smartlist_add_linecpy(diff, area, "+0,4d"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an ed command was missing a line number"); + + smartlist_clear(diff); + + /* range duplication */ + smartlist_add_linecpy(diff, area, "0,4d,5d"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an ed command longer than one char was " + "found"); + + smartlist_clear(diff); + + /* space before command */ + smartlist_add_linecpy(diff, area, "0,4 d"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an ed command longer than one char was " + "found"); + + smartlist_clear(diff); + + /* space inside number */ + smartlist_add_linecpy(diff, area, "0,4 5d"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an ed command longer than one char was " + "found"); + + smartlist_clear(diff); + /* Test appending text, 'a'. */ consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); @@ -775,7 +846,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test changing text, 'c'. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n1, 2c\nM\n.\n", area); + consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(5, OP_EQ, smartlist_len(cons2)); diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c new file mode 100644 index 0000000000..2fb7dd2dfb --- /dev/null +++ b/src/test/test_consdiffmgr.c @@ -0,0 +1,888 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CONSDIFFMGR_PRIVATE + +#include "or.h" +#include "config.h" +#include "conscache.h" +#include "consdiff.h" +#include "consdiffmgr.h" +#include "cpuworker.h" +#include "networkstatus.h" +#include "workqueue.h" + +#include "test.h" +#include "log_test_helpers.h" + +// ============================== Setup/teardown the consdiffmgr +// These functions get run before/after each test in this module + +static void * +consdiffmgr_test_setup(const struct testcase_t *arg) +{ + (void)arg; + char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm")); + tor_free(get_options_mutable()->DataDirectory); + get_options_mutable()->DataDirectory = ddir_fname; // now owns the pointer. + check_private_dir(ddir_fname, CPD_CREATE, NULL); + + consdiff_cfg_t consdiff_cfg = { 7200, 300 }; + consdiffmgr_configure(&consdiff_cfg); + return (void *)1; // must return something non-null. +} +static int +consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore) +{ + (void)arg; + (void)ignore; + consdiffmgr_free_all(); + return 1; +} +static struct testcase_setup_t setup_diffmgr = { + consdiffmgr_test_setup, + consdiffmgr_test_teardown +}; + +// ============================== NS faking functions +// These functions are for making quick fake consensus objects and +// strings that are just good enough for consdiff and consdiffmgr. + +static networkstatus_t * +fake_ns_new(consensus_flavor_t flav, time_t valid_after) +{ + networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t)); + ns->type = NS_TYPE_CONSENSUS; + ns->flavor = flav; + ns->valid_after = valid_after; + return ns; +} + +static char * +fake_ns_body_new(consensus_flavor_t flav, time_t valid_after) +{ + const char *flavor_string = flav == FLAV_NS ? "" : " microdesc"; + char valid_after_string[ISO_TIME_LEN+1]; + + format_iso_time(valid_after_string, valid_after); + char *random_stuff = crypto_random_hostname(3, 25, "junk ", ""); + + char *consensus; + tor_asprintf(&consensus, + "network-status-version 3%s\n" + "vote-status consensus\n" + "valid-after %s\n" + "r name ccccccccccccccccc etc\nsample\n" + "r name eeeeeeeeeeeeeeeee etc\nbar\n" + "%s\n", + flavor_string, + valid_after_string, + random_stuff); + tor_free(random_stuff); + return consensus; +} + +// ============================== Cpuworker mocking code +// These mocking functions and types capture the cpuworker calls +// so we can inspect them and run them in the main thread. +static smartlist_t *fake_cpuworker_queue = NULL; +typedef struct fake_work_queue_ent_t { + enum workqueue_reply_t (*fn)(void *, void *); + void (*reply_fn)(void *); + void *arg; +} fake_work_queue_ent_t; +static struct workqueue_entry_s * +mock_cpuworker_queue_work(enum workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg) +{ + if (! fake_cpuworker_queue) + fake_cpuworker_queue = smartlist_new(); + + fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent)); + ent->fn = fn; + ent->reply_fn = reply_fn; + ent->arg = arg; + smartlist_add(fake_cpuworker_queue, ent); + return (struct workqueue_entry_s *)ent; +} +static int +mock_cpuworker_run_work(void) +{ + if (! fake_cpuworker_queue) + return 0; + SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, { + enum workqueue_reply_t r = ent->fn(NULL, ent->arg); + if (r != WQ_RPL_REPLY) + return -1; + }); + return 0; +} +static void +mock_cpuworker_handle_replies(void) +{ + if (! fake_cpuworker_queue) + return; + SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, { + ent->reply_fn(ent->arg); + tor_free(ent); + }); + smartlist_free(fake_cpuworker_queue); + fake_cpuworker_queue = NULL; +} + +// ============================== Other helpers + +static consdiff_status_t +lookup_diff_from(consensus_cache_entry_t **out, + consensus_flavor_t flav, + const char *str1) +{ + uint8_t digest[DIGEST256_LEN]; + crypto_digest256((char*)digest, str1, strlen(str1), DIGEST_SHA3_256); + return consdiffmgr_find_diff_from(out, flav, + DIGEST_SHA3_256, digest, sizeof(digest)); +} + +static int +lookup_apply_and_verify_diff(consensus_flavor_t flav, + const char *str1, + const char *str2) +{ + char *diff_string = NULL; + consensus_cache_entry_t *ent = NULL; + consdiff_status_t status = lookup_diff_from(&ent, flav, str1); + if (ent == NULL || status != CONSDIFF_AVAILABLE) + return -1; + + consensus_cache_entry_incref(ent); + size_t size; + const uint8_t *body; + int r = consensus_cache_entry_get_body(ent, &body, &size); + if (r == 0) + diff_string = tor_memdup_nulterm(body, size); + consensus_cache_entry_decref(ent); + if (diff_string == NULL) + return -1; + + char *applied = consensus_diff_apply(str1, diff_string); + tor_free(diff_string); + if (applied == NULL) + return -1; + + int match = !strcmp(applied, str2); + tor_free(applied); + return match ? 0 : -1; +} + +static void +cdm_reload(void) +{ + consdiffmgr_free_all(); + cdm_cache_get(); + consdiffmgr_rescan(); +} + +// ============================== Beginning of tests + +#if 0 +static int got_failure = 0; +static void +got_assertion_failure(void) +{ + ++got_failure; +} + +/* XXXX This test won't work, because there is currently no way to actually + * XXXX capture a real assertion failure. */ +static void +test_consdiffmgr_init_failure(void *arg) +{ + (void)arg; + // Capture assertions and bugs. + + /* As in ...test_setup, but do not create the datadir. The missing directory + * will cause a failure. */ + char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm")); + tor_free(get_options_mutable()->DataDirectory); + get_options_mutable()->DataDirectory = ddir_fname; // now owns the pointer. + + consdiff_cfg_t consdiff_cfg = { 7200, 300 }; + + tor_set_failed_assertion_callback(got_assertion_failure); + tor_capture_bugs_(1); + consdiffmgr_configure(&consdiff_cfg); // This should fail. + tt_int_op(got_failure, OP_EQ, 1); + const smartlist_t *bugs = tor_get_captured_bug_log_(); + tt_int_op(smartlist_len(bugs), OP_EQ, 1); + + done: + tor_end_capture_bugs_(); +} +#endif + +static void +test_consdiffmgr_sha3_helper(void *arg) +{ + (void) arg; + consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier + config_line_t *lines = NULL; + char *mem_op_hex_tmp = NULL; + config_line_prepend(&lines, "good-sha", + "F00DF00DF00DF00DF00DF00DF00DF00D" + "F00DF00DF00DF00DF00DF00DF00DF00D"); + config_line_prepend(&lines, "short-sha", + "F00DF00DF00DF00DF00DF00DF00DF00D" + "F00DF00DF00DF00DF00DF00DF00DF0"); + config_line_prepend(&lines, "long-sha", + "F00DF00DF00DF00DF00DF00DF00DF00D" + "F00DF00DF00DF00DF00DF00DF00DF00DF00D"); + config_line_prepend(&lines, "not-sha", + "F00DF00DF00DF00DF00DF00DF00DF00D" + "F00DF00DF00DF00DF00DF00DF00DXXXX"); + consensus_cache_entry_t *ent = + consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8); + + uint8_t buf[DIGEST256_LEN]; + tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha")); + tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha")); + test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D" + "F00DF00DF00DF00DF00DF00DF00DF00D"); + + tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha")); + tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha")); + tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha")); + tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha")); + + done: + consensus_cache_entry_decref(ent); + config_free_lines(lines); + tor_free(mem_op_hex_tmp); +} + +static void +test_consdiffmgr_add(void *arg) +{ + (void) arg; + time_t now = approx_time(); + + consensus_cache_entry_t *ent = NULL; + networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); + const char *dummy = "foo"; + int r = consdiffmgr_add_consensus(dummy, ns_tmp); + tt_int_op(r, OP_EQ, 0); + + /* If we add it again, it won't work */ + setup_capture_of_logs(LOG_INFO); + dummy = "bar"; + r = consdiffmgr_add_consensus(dummy, ns_tmp); + tt_int_op(r, OP_EQ, -1); + expect_single_log_msg_containing("We already have a copy of that " + "consensus"); + mock_clean_saved_logs(); + + /* But it will work fine if the flavor is different */ + dummy = "baz"; + ns_tmp->flavor = FLAV_MICRODESC; + r = consdiffmgr_add_consensus(dummy, ns_tmp); + tt_int_op(r, OP_EQ, 0); + + /* And it will work fine if the time is different */ + dummy = "quux"; + ns_tmp->flavor = FLAV_NS; + ns_tmp->valid_after = now - 60; + r = consdiffmgr_add_consensus(dummy, ns_tmp); + tt_int_op(r, OP_EQ, 0); + + /* If we add one a long long time ago, it will fail. */ + dummy = "xyzzy"; + ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */ + r = consdiffmgr_add_consensus(dummy, ns_tmp); + tt_int_op(r, OP_EQ, -1); + expect_single_log_msg_containing("it's too old."); + + /* Try looking up a consensuses. */ + ent = cdm_cache_lookup_consensus(FLAV_NS, now-60); + tt_assert(ent); + consensus_cache_entry_incref(ent); + size_t s; + const uint8_t *body; + r = consensus_cache_entry_get_body(ent, &body, &s); + tt_int_op(r, OP_EQ, 0); + tt_int_op(s, OP_EQ, 4); + tt_mem_op(body, OP_EQ, "quux", 4); + + /* Try looking up another entry, but fail */ + tt_assert(NULL == cdm_cache_lookup_consensus(FLAV_MICRODESC, now-60)); + tt_assert(NULL == cdm_cache_lookup_consensus(FLAV_NS, now-61)); + + done: + networkstatus_vote_free(ns_tmp); + teardown_capture_of_logs(); + consensus_cache_entry_decref(ent); +} + +static void +test_consdiffmgr_make_diffs(void *arg) +{ + (void)arg; + networkstatus_t *ns = NULL; + char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL; + char *applied = NULL, *diff_text = NULL; + time_t now = approx_time(); + int r; + consensus_cache_entry_t *diff = NULL; + uint8_t md_ns_sha3[DIGEST256_LEN]; + consdiff_status_t diff_status; + + MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); + + // Try rescan with no consensuses: shouldn't crash or queue work. + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); + + // Make two consensuses, 1 hour sec ago. + ns = fake_ns_new(FLAV_NS, now-3600); + ns_body = fake_ns_body_new(FLAV_NS, now-3600); + r = consdiffmgr_add_consensus(ns_body, ns); + networkstatus_vote_free(ns); + tor_free(ns_body); + tt_int_op(r, OP_EQ, 0); + + ns = fake_ns_new(FLAV_MICRODESC, now-3600); + md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); + r = consdiffmgr_add_consensus(md_ns_body, ns); + crypto_digest256((char*)md_ns_sha3, md_ns_body, strlen(md_ns_body), + DIGEST_SHA3_256); + networkstatus_vote_free(ns); + tt_int_op(r, OP_EQ, 0); + + // No diffs will be generated. + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); + + // Add a MD consensus from 45 minutes ago. This should cause one diff + // worth of work to get queued. + ns = fake_ns_new(FLAV_MICRODESC, now-45*60); + md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60); + r = consdiffmgr_add_consensus(md_ns_body_2, ns); + networkstatus_vote_free(ns); + tt_int_op(r, OP_EQ, 0); + + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); + tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue)); + diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC, + DIGEST_SHA3_256, + md_ns_sha3, DIGEST256_LEN); + tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status); + + // Now run that process and get the diff. + r = mock_cpuworker_run_work(); + tt_int_op(r, OP_EQ, 0); + mock_cpuworker_handle_replies(); + + // At this point we should be able to get that diff. + diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC, + DIGEST_SHA3_256, + md_ns_sha3, DIGEST256_LEN); + tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status); + tt_assert(diff); + + /* Make sure applying the diff actually works */ + const uint8_t *diff_body; + size_t diff_size; + r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size); + tt_int_op(r, OP_EQ, 0); + diff_text = tor_memdup_nulterm(diff_body, diff_size); + applied = consensus_diff_apply(md_ns_body, diff_text); + tt_assert(applied); + tt_str_op(applied, OP_EQ, md_ns_body_2); + + /* Rescan again: no more work to do. */ + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); + + done: + tor_free(md_ns_body); + tor_free(md_ns_body_2); + tor_free(diff_text); + tor_free(applied); +} + +static void +test_consdiffmgr_diff_rules(void *arg) +{ + (void)arg; +#define N 6 + char *md_body[N], *ns_body[N]; + networkstatus_t *md_ns[N], *ns_ns[N]; + uint8_t md_ns_sha3[N][DIGEST256_LEN], ns_ns_sha3[N][DIGEST256_LEN]; + int i; + + MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); + + /* Create a bunch of consensus things at 15-second intervals. */ + time_t start = approx_time() - 120; + for (i = 0; i < N; ++i) { + time_t when = start + i * 15; + md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when); + ns_body[i] = fake_ns_body_new(FLAV_NS, when); + md_ns[i] = fake_ns_new(FLAV_MICRODESC, when); + ns_ns[i] = fake_ns_new(FLAV_NS, when); + crypto_digest256((char *)md_ns_sha3[i], md_body[i], strlen(md_body[i]), + DIGEST_SHA3_256); + crypto_digest256((char *)ns_ns_sha3[i], ns_body[i], strlen(ns_body[i]), + DIGEST_SHA3_256); + } + + /* For the MD consensuses: add 4 of them, and make sure that + * diffs are created to one consensus (the most recent) only. */ + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4])); + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); + tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue)); + tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); + mock_cpuworker_handle_replies(); + tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); + + /* For the NS consensuses: add 3, generate, and add one older one and + * make sure that older one is the only one whose diff is generated */ + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5])); + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); + tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue)); + tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); + mock_cpuworker_handle_replies(); + + /* At this point, we should actually have working diffs! */ + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5])); + + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4])); + + /* Self-to-self diff won't be present */ + consensus_cache_entry_t *ent; + tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, + lookup_diff_from(&ent, FLAV_NS, ns_body[5])); + /* No diff from 2 has been added yet */ + tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, + lookup_diff_from(&ent, FLAV_NS, ns_body[2])); + /* No diff arriving at old things. */ + tt_int_op(-1, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2])); + /* No backwards diff */ + tt_int_op(-1, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3])); + + /* Now, an update: add number 2 and make sure it's the only one whose diff + * is regenerated. */ + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2])); + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); + tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue)); + tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); + mock_cpuworker_handle_replies(); + + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5])); + + /* Finally: reload, and make sure that the information is still indexed */ + cdm_reload(); + + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5])); + + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4])); + + done: + for (i = 0; i < N; ++i) { + tor_free(md_body[i]); + tor_free(ns_body[i]); + networkstatus_vote_free(md_ns[i]); + networkstatus_vote_free(ns_ns[i]); + } + UNMOCK(cpuworker_queue_work); +#undef N +} + +static void +test_consdiffmgr_diff_failure(void *arg) +{ + (void)arg; + MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); + + /* We're going to make sure that if we have a bogus request where + * we can't actually compute a diff, the world must not end. */ + networkstatus_t *ns1 = NULL; + networkstatus_t *ns2 = NULL; + int r; + + ns1 = fake_ns_new(FLAV_NS, approx_time()-100); + ns2 = fake_ns_new(FLAV_NS, approx_time()-50); + r = consdiffmgr_add_consensus("foo bar baz\n", ns1); + tt_int_op(r, OP_EQ, 0); + // We refuse to compute a diff to or from a line holding only a single dot. + // We can add it here, though. + r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2); + tt_int_op(r, OP_EQ, 0); + + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); + setup_capture_of_logs(LOG_WARN); + tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue)); + tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); + expect_single_log_msg_containing("one of the lines to be added is \".\"."); + mock_clean_saved_logs(); + mock_cpuworker_handle_replies(); + expect_single_log_msg_containing("Worker was unable to compute consensus " + "diff from "); + + /* Make sure the diff is not present */ + consensus_cache_entry_t *ent; + tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, + lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n")); + + done: + teardown_capture_of_logs(); + UNMOCK(cpuworker_queue_work); + networkstatus_vote_free(ns1); + networkstatus_vote_free(ns2); +} + +static void +test_consdiffmgr_diff_pending(void *arg) +{ +#define N 3 + (void)arg; + char *md_body[N]; + networkstatus_t *md_ns[N]; + time_t start = approx_time() - 120; + int i; + for (i = 0; i < N; ++i) { + time_t when = start + i * 30; + md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when); + md_ns[i] = fake_ns_new(FLAV_MICRODESC, when); + } + + MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); + + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2])); + /* Make a diff */ + consdiffmgr_rescan(); + tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue)); + + /* Look it up. Is it pending? */ + consensus_cache_entry_t *ent = NULL; + consdiff_status_t diff_status; + diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]); + tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status); + tt_ptr_op(ent, OP_EQ, NULL); + + /* Add another old consensus. only one new diff should launch! */ + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0])); + consdiffmgr_rescan(); + tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue)); + + tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); + mock_cpuworker_handle_replies(); + + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2])); + + done: + UNMOCK(cpuworker_queue_work); + for (i = 0; i < N; ++i) { + tor_free(md_body[i]); + networkstatus_vote_free(md_ns[i]); + } +#undef N +} + +static void +test_consdiffmgr_cleanup_old(void *arg) +{ + (void)arg; + config_line_t *labels = NULL; + consensus_cache_entry_t *ent = NULL; + consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier + + /* This item will be will be cleanable because it has a valid-after + * time far in the past. */ + config_line_prepend(&labels, "document-type", "confribble-blarg"); + config_line_prepend(&labels, "consensus-valid-after", + "1980-10-10T10:10:10"); + ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3); + tt_assert(ent); + consensus_cache_entry_decref(ent); + + setup_capture_of_logs(LOG_DEBUG); + tt_int_op(1, OP_EQ, consdiffmgr_cleanup()); + expect_log_msg_containing("Deleting entry because its consensus-valid-" + "after value (1980-10-10T10:10:10) was too old"); + + done: + teardown_capture_of_logs(); + config_free_lines(labels); +} + +static void +test_consdiffmgr_cleanup_bad_valid_after(void *arg) +{ + /* This will seem cleanable, but isn't, because its valid-after time is + * misformed. */ + + (void)arg; + config_line_t *labels = NULL; + consensus_cache_entry_t *ent = NULL; + consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier + + config_line_prepend(&labels, "document-type", "consensus"); + config_line_prepend(&labels, "consensus-valid-after", + "whan that aprille with his shoures soote"); // (~1385?) + ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3); + tt_assert(ent); + consensus_cache_entry_decref(ent); + + setup_capture_of_logs(LOG_DEBUG); + tt_int_op(0, OP_EQ, consdiffmgr_cleanup()); + expect_log_msg_containing("Ignoring entry because its consensus-valid-" + "after value (\"whan that aprille with his " + "shoures soote\") was unparseable"); + + done: + teardown_capture_of_logs(); + config_free_lines(labels); +} + +static void +test_consdiffmgr_cleanup_no_valid_after(void *arg) +{ + (void)arg; + config_line_t *labels = NULL; + consensus_cache_entry_t *ent = NULL; + consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier + + /* This item will be will be uncleanable because it has no recognized + * valid-after. */ + config_line_prepend(&labels, "document-type", "consensus"); + config_line_prepend(&labels, "confrooble-voolid-oofter", + "2010-10-10T09:08:07"); + ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3); + tt_assert(ent); + consensus_cache_entry_decref(ent); + + setup_capture_of_logs(LOG_DEBUG); + tt_int_op(0, OP_EQ, consdiffmgr_cleanup()); + expect_log_msg_containing("Ignoring entry because it had no consensus-" + "valid-after label"); + + done: + teardown_capture_of_logs(); + config_free_lines(labels); +} + +static void +test_consdiffmgr_cleanup_old_diffs(void *arg) +{ + (void)arg; +#define N 4 + char *md_body[N]; + networkstatus_t *md_ns[N]; + uint8_t md_ns_sha3[N][DIGEST256_LEN]; + int i; + consensus_cache_entry_t *hold_ent = NULL, *ent; + + /* Make sure that the cleanup function removes diffs to the not-most-recent + * consensus. */ + + MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); + + /* Create a bunch of consensus things at 15-second intervals. */ + time_t start = approx_time() - 120; + for (i = 0; i < N; ++i) { + time_t when = start + i * 15; + md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when); + md_ns[i] = fake_ns_new(FLAV_MICRODESC, when); + crypto_digest256((char *)md_ns_sha3[i], md_body[i], strlen(md_body[i]), + DIGEST_SHA3_256); + } + + /* add the first 3. */ + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1])); + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2])); + /* Make diffs. */ + consdiffmgr_rescan(); + tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); + tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue)); + tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); + mock_cpuworker_handle_replies(); + tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); + + /* Nothing is deletable now */ + tt_int_op(0, OP_EQ, consdiffmgr_cleanup()); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2])); + tt_int_op(0, OP_EQ, + lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2])); + + tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, + lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1])); + consensus_cache_entry_incref(hold_ent); // incref, so it is preserved. + + /* Now add an even-more-recent consensus; this should make all previous + * diffs deletable */ + tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3])); + tt_int_op(2, OP_EQ, consdiffmgr_cleanup()); + + tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, + lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0])); + /* This one is marked deletable but still in the hashtable */ + tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, + lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1])); + tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, + lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2])); + + /* Everything should be valid at this point */ + tt_int_op(0, OP_EQ, consdiffmgr_validate()); + + /* And if we recan NOW, we'll purge the hashtable of the entries, + * and launch attempts to generate new ones */ + consdiffmgr_rescan(); + tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, + lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0])); + tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, + lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1])); + tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, + lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2])); + + /* We're still holding on to this, though, so we can still map it! */ + const uint8_t *t1 = NULL; + size_t s; + int r = consensus_cache_entry_get_body(hold_ent, &t1, &s); + tt_int_op(r, OP_EQ, 0); + tt_assert(t1); + + done: + for (i = 0; i < N; ++i) { + tor_free(md_body[i]); + networkstatus_vote_free(md_ns[i]); + } + consensus_cache_entry_decref(hold_ent); + UNMOCK(cpuworker_queue_work); +#undef N +} + +static void +test_consdiffmgr_validate(void *arg) +{ + (void)arg; + config_line_t *lines = NULL; + consensus_cache_entry_t *ent = NULL; + consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier + smartlist_t *vals = smartlist_new(); + + /* Put these: objects in the cache: one with a good sha3, one with bad sha3, + * one with a wrong sha3, and one with no sha3. */ + config_line_prepend(&lines, "id", "wrong sha3"); + config_line_prepend(&lines, "sha3-digest", + "F00DF00DF00DF00DF00DF00DF00DF00D" + "F00DF00DF00DF00DF00DF00DF00DF00D"); + ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8); + consensus_cache_entry_decref(ent); + config_free_lines(lines); + lines = NULL; + + config_line_prepend(&lines, "id", "bad sha3"); + config_line_prepend(&lines, "sha3-digest", + "now is the winter of our dicotheque"); + ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8); + consensus_cache_entry_decref(ent); + config_free_lines(lines); + lines = NULL; + + config_line_prepend(&lines, "id", "no sha3"); + ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8); + consensus_cache_entry_decref(ent); + config_free_lines(lines); + lines = NULL; + + config_line_prepend(&lines, "id", "good sha3"); + config_line_prepend(&lines, "sha3-digest", + "8d8b1998616cd6b4c4055da8d38728dc" + "93c758d4131a53c7d81aa6337dee1c05"); + ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8); + consensus_cache_entry_decref(ent); + config_free_lines(lines); + lines = NULL; + + cdm_reload(); + cache = cdm_cache_get(); + tt_int_op(1, OP_EQ, consdiffmgr_validate()); + + consensus_cache_find_all(vals, cache, "id", "good sha3"); + tt_int_op(smartlist_len(vals), OP_EQ, 1); + smartlist_clear(vals); + + consensus_cache_find_all(vals, cache, "id", "no sha3"); + tt_int_op(smartlist_len(vals), OP_EQ, 1); + smartlist_clear(vals); + + consensus_cache_find_all(vals, cache, "id", "wrong sha3"); + tt_int_op(smartlist_len(vals), OP_EQ, 0); + consensus_cache_find_all(vals, cache, "id", "bad sha3"); + tt_int_op(smartlist_len(vals), OP_EQ, 0); + + done: + smartlist_free(vals); +} + +#define TEST(name) \ + { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL } + +struct testcase_t consdiffmgr_tests[] = { +#if 0 + { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL }, +#endif + TEST(sha3_helper), + TEST(add), + TEST(make_diffs), + TEST(diff_rules), + TEST(diff_failure), + TEST(diff_pending), + TEST(cleanup_old), + TEST(cleanup_bad_valid_after), + TEST(cleanup_no_valid_after), + TEST(cleanup_old_diffs), + TEST(validate), + + // XXXX Test: non-cacheing cases of replyfn(). + + END_OF_TESTCASES +}; + diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 048d773475..a9d9cba7df 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -7,6 +7,7 @@ #include <math.h> #define CONFIG_PRIVATE +#define CONTROL_PRIVATE #define DIRSERV_PRIVATE #define DIRVOTE_PRIVATE #define ROUTER_PRIVATE @@ -19,6 +20,7 @@ #include "or.h" #include "confparse.h" #include "config.h" +#include "control.h" #include "crypto_ed25519.h" #include "directory.h" #include "dirserv.h" @@ -910,6 +912,23 @@ mock_get_by_ei_desc_digest(const char *d) } } +static signed_descriptor_t * +mock_ei_get_by_ei_digest(const char *d) +{ + char hex[HEX_DIGEST_LEN+1]; + base16_encode(hex, sizeof(hex), d, DIGEST_LEN); + signed_descriptor_t *sd = &sd_ei_minimal; + + if (!strcmp(hex, "11E0EDF526950739F7769810FCACAB8C882FAEEE")) { + sd->signed_descriptor_body = (char *)EX_EI_MINIMAL; + sd->signed_descriptor_len = sizeof(EX_EI_MINIMAL); + sd->annotations_len = 0; + sd->saved_location = SAVED_NOWHERE; + return sd; + } + return NULL; +} + static smartlist_t *mock_ei_insert_list = NULL; static was_router_added_t mock_ei_insert(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible) @@ -999,6 +1018,37 @@ test_dir_load_extrainfo(void *arg) } static void +test_dir_getinfo_extra(void *arg) +{ + int r; + char *answer = NULL; + const char *errmsg = NULL; + + (void)arg; + MOCK(extrainfo_get_by_descriptor_digest, mock_ei_get_by_ei_digest); + r = getinfo_helper_dir(NULL, "extra-info/digest/" + "11E0EDF526950739F7769810FCACAB8C882FAEEE", &answer, + &errmsg); + tt_int_op(0, OP_EQ, r); + tt_ptr_op(NULL, OP_EQ, errmsg); + tt_str_op(answer, OP_EQ, EX_EI_MINIMAL); + tor_free(answer); + + answer = NULL; + r = getinfo_helper_dir(NULL, "extra-info/digest/" + "NOTAVALIDHEXSTRINGNOTAVALIDHEXSTRINGNOTA", &answer, + &errmsg); + tt_int_op(0, OP_EQ, r); + /* getinfo_helper_dir() should maybe return an error here but doesn't */ + tt_ptr_op(NULL, OP_EQ, errmsg); + /* In any case, there should be no answer for an invalid hex string. */ + tt_ptr_op(NULL, OP_EQ, answer); + + done: + UNMOCK(extrainfo_get_by_descriptor_digest); +} + +static void test_dir_versions(void *arg) { tor_version_t ver1; @@ -5946,6 +5996,7 @@ struct testcase_t dir_tests[] = { DIR(parse_router_list, TT_FORK), DIR(load_routers, TT_FORK), DIR(load_extrainfo, TT_FORK), + DIR(getinfo_extra, 0), DIR_LEGACY(versions), DIR_LEGACY(fp_pairs), DIR(split_fps, 0), diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index cfda314693..392fa4dde0 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -14,6 +14,7 @@ #include "connection.h" #include "directory.h" #include "test.h" +#include "compress.h" #include "connection.h" #include "rendcommon.h" #include "rendcache.h" @@ -28,7 +29,6 @@ #include "networkstatus.h" #include "geoip.h" #include "dirserv.h" -#include "torgzip.h" #include "dirvote.h" #include "log_test_helpers.h" @@ -1832,8 +1832,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data) comp_body_used); tt_int_op(ZLIB_METHOD, OP_EQ, compression); - tor_gzip_uncompress(&body, &body_used, comp_body, comp_body_used, - compression, 0, LOG_PROTOCOL_WARN); + tor_uncompress(&body, &body_used, comp_body, comp_body_used, + compression, 0, LOG_PROTOCOL_WARN); tt_str_op(NETWORK_STATUS, OP_EQ, body); tt_int_op(strlen(NETWORK_STATUS), OP_EQ, body_used); diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c index 7bcd5685c3..ef5cb46567 100644 --- a/src/test/test_hs_intropoint.c +++ b/src/test/test_hs_intropoint.c @@ -777,11 +777,11 @@ test_received_introduce1_handling(void *arg) { cell = helper_create_introduce1_cell(); ssize_t request_len = trn_cell_introduce1_encoded_len(cell); - tt_size_op(request_len, OP_GT, 0); + tt_int_op((int)request_len, OP_GT, 0); request = tor_malloc_zero(request_len); ssize_t encoded_len = trn_cell_introduce1_encode(request, request_len, cell); - tt_size_op(encoded_len, OP_GT, 0); + tt_int_op((int)encoded_len, OP_GT, 0); circ = helper_create_intro_circuit(); or_circuit_t *service_circ = helper_create_intro_circuit(); @@ -812,7 +812,7 @@ test_received_introduce1_handling(void *arg) request = tor_malloc_zero(request_len + 256); ssize_t encoded_len = trn_cell_introduce1_encode(request, request_len, cell); - tt_size_op(encoded_len, OP_GT, 0); + tt_int_op((int)encoded_len, OP_GT, 0); circ = helper_create_intro_circuit(); or_circuit_t *service_circ = helper_create_intro_circuit(); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index ec9cc4579b..fcfb3b992d 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -6,6 +6,7 @@ * \brief Test hidden service functionality. */ +#define HS_COMMON_PRIVATE #define HS_SERVICE_PRIVATE #define HS_INTROPOINT_PRIVATE @@ -14,6 +15,7 @@ #include "crypto.h" #include "hs/cell_establish_intro.h" +#include "hs_common.h" #include "hs_service.h" #include "hs_intropoint.h" @@ -196,12 +198,52 @@ test_hs_ntor(void *arg) ; } +/** Test that our HS time period calculation functions work properly */ +static void +test_time_period(void *arg) +{ + (void) arg; + uint64_t tn; + int retval; + time_t fake_time; + + /* Let's do the example in prop224 section [TIME-PERIODS] */ + retval = parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC", + &fake_time); + tt_int_op(retval, ==, 0); + + /* Check that the time period number is right */ + tn = get_time_period_num(fake_time); + tt_u64_op(tn, ==, 16903); + + /* Increase current time to 11:59:59 UTC and check that the time period + number is still the same */ + fake_time += 3599; + tn = get_time_period_num(fake_time); + tt_u64_op(tn, ==, 16903); + + /* Now take time to 12:00:00 UTC and check that the time period rotated */ + fake_time += 1; + tn = get_time_period_num(fake_time); + tt_u64_op(tn, ==, 16904); + + /* Now also check our hs_get_next_time_period_num() function */ + tn = hs_get_next_time_period_num(fake_time); + tt_u64_op(tn, ==, 16905); + + done: + ; +} + struct testcase_t hs_service_tests[] = { { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK, NULL, NULL }, { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK, NULL, NULL }, - { "hs_ntor", test_hs_ntor, TT_FORK, NULL, NULL }, + { "hs_ntor", test_hs_ntor, TT_FORK, + NULL, NULL }, + { "time_period", test_time_period, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_util.c b/src/test/test_util.c index 203f9dd1c4..d47475a24c 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2249,46 +2249,49 @@ test_util_gzip(void *arg) char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; const char *ccp2; size_t len1, len2; - tor_zlib_state_t *state = NULL; + tor_compress_state_t *state = NULL; (void)arg; + tt_assert(tor_compress_supports_method(GZIP_METHOD)); + tt_assert(tor_compress_supports_method(ZLIB_METHOD)); + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - tt_assert(buf2); - tt_assert(len1 < strlen(buf1)); - tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, GZIP_METHOD); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - tt_assert(buf3); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); tor_free(buf2); tor_free(buf3); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); tt_assert(buf2); - tt_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, ZLIB_METHOD); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); - tt_assert(buf3); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); /* Check whether we can uncompress concatenated, compressed strings. */ tor_free(buf3); buf2 = tor_reallocarray(buf2, len1, 2); memcpy(buf2+len1, buf2, len1); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); - tt_int_op((strlen(buf1)+1)*2,OP_EQ, len2); - tt_mem_op(buf3,OP_EQ, + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + ZLIB_METHOD, 1, LOG_INFO)); + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", (strlen(buf1)+1)*2); @@ -2300,53 +2303,53 @@ test_util_gzip(void *arg) /* Check whether we can uncompress partial strings. */ buf1 = tor_strdup("String with low redundancy that won't be compressed much."); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - tt_assert(len1>16); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + tt_int_op(len1, OP_GT, 16); /* when we allow an incomplete string, we should succeed.*/ - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 0, LOG_INFO)); tt_assert(len2 > 5); buf3[len2]='\0'; tt_assert(!strcmpstart(buf1, buf3)); /* when we demand a complete string, this must fail. */ tor_free(buf3); - tt_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); - tt_assert(!buf3); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(buf3 == NULL); /* Now, try streaming compression. */ tor_free(buf1); tor_free(buf2); tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); tt_assert(state); cp1 = buf1 = tor_malloc(1024); len1 = 1024; ccp2 = "ABCDEFGHIJABCDEFGHIJ"; len2 = 21; - tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); - tt_int_op(0,OP_EQ, len2); /* Make sure we compressed it all. */ + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ tt_assert(cp1 > buf1); len2 = 0; cp2 = cp1; - tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); - tt_int_op(0,OP_EQ, len2); + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); + tt_int_op(0, OP_EQ, len2); tt_assert(cp1 > cp2); /* Make sure we really added something. */ - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + ZLIB_METHOD, 1, LOG_WARN)); /* Make sure it compressed right. */ tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); - tt_int_op(21,OP_EQ, len2); + tt_int_op(21, OP_EQ, len2); done: if (state) - tor_zlib_free(state); + tor_compress_free(state); tor_free(buf2); tor_free(buf3); tor_free(buf1); @@ -2364,44 +2367,268 @@ test_util_gzip_compression_bomb(void *arg) char *one_mb = tor_malloc_zero(one_million); char *result = NULL; size_t result_len = 0; - tor_zlib_state_t *state = NULL; + tor_compress_state_t *state = NULL; /* Make sure we can't produce a compression bomb */ setup_full_capture_of_logs(LOG_WARN); - tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len, - one_mb, one_million, - ZLIB_METHOD)); + tt_int_op(-1, OP_EQ, tor_compress(&result, &result_len, + one_mb, one_million, + ZLIB_METHOD)); expect_single_log_msg_containing( "We compressed something and got an insanely high " "compression factor; other Tors would think this " - "was a zlib bomb."); + "was a compression bomb."); teardown_capture_of_logs(); /* Here's a compression bomb that we made manually. */ const char compression_bomb[1039] = { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2, 0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ }; - tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len, - compression_bomb, 1039, - ZLIB_METHOD, 0, LOG_WARN)); + tt_int_op(-1, OP_EQ, tor_uncompress(&result, &result_len, + compression_bomb, 1039, + ZLIB_METHOD, 0, LOG_WARN)); /* Now try streaming that. */ - state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); - tor_zlib_output_t r; + state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + tor_compress_output_t r; const char *inp = compression_bomb; size_t inlen = 1039; do { char *outp = one_mb; size_t outleft = 4096; /* small on purpose */ - r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0); + r = tor_compress_process(state, &outp, &outleft, &inp, &inlen, 0); tt_int_op(inlen, OP_NE, 0); - } while (r == TOR_ZLIB_BUF_FULL); + } while (r == TOR_COMPRESS_BUFFER_FULL); - tt_int_op(r, OP_EQ, TOR_ZLIB_ERR); + tt_int_op(r, OP_EQ, TOR_COMPRESS_ERROR); done: tor_free(one_mb); - tor_zlib_free(state); + tor_compress_free(state); +} + +static void +test_util_lzma(void *arg) +{ +#ifdef HAVE_LZMA + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_compress_state_t *state = NULL; + + (void)arg; + tt_assert(tor_compress_supports_method(LZMA_METHOD)); + + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + LZMA_METHOD)); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, LZMA_METHOD); + + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + LZMA_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + +#if 0 + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_reallocarray(buf2, len1, 2); + memcpy(buf2+len1, buf2, len1); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + LZMA_METHOD, 1, LOG_INFO)); + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + LZMA_METHOD)); + tt_int_op(len1, OP_GT, 16); + /* when we allow an incomplete string, we should succeed.*/ + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + LZMA_METHOD, 0, LOG_INFO)); + tt_assert(len2 > 5); + buf3[len2]='\0'; + tt_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + LZMA_METHOD, 1, LOG_INFO)); + tt_assert(buf3 == NULL); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); +#endif + + /* Now, try streaming compression. */ + state = tor_compress_new(1, LZMA_METHOD, HIGH_COMPRESSION); + tt_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ + tt_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); + tt_int_op(0, OP_EQ, len2); + tt_assert(cp1 > cp2); /* Make sure we really added something. */ + + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + LZMA_METHOD, 1, LOG_WARN)); + /* Make sure it compressed right. */ + tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); + tt_int_op(21, OP_EQ, len2); + + done: + if (state) + tor_compress_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +#else + (void)arg; + tt_assert(! tor_compress_supports_method(LZMA_METHOD)); + + done: + ; +#endif // HAVE_LZMA. +} + +static void +test_util_zstd(void *arg) +{ +#ifdef HAVE_ZSTD + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_compress_state_t *state = NULL; + + (void)arg; + tt_assert(tor_compress_supports_method(ZSTD_METHOD)); + + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZSTD_METHOD)); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, ZSTD_METHOD); + + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + ZSTD_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + +#if 0 + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_reallocarray(buf2, len1, 2); + memcpy(buf2+len1, buf2, len1); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + ZSTD_METHOD, 1, LOG_INFO)); + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZSTD_METHOD)); + tt_int_op(len1, OP_GT, 16); + /* when we allow an incomplete string, we should succeed.*/ + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + ZSTD_METHOD, 0, LOG_INFO)); + tt_assert(len2 > 5); + buf3[len2]='\0'; + tt_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + ZSTD_METHOD, 1, LOG_INFO)); + tt_assert(buf3 == NULL); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); +#endif + + /* Now, try streaming compression. */ + state = tor_compress_new(1, ZSTD_METHOD, HIGH_COMPRESSION); + tt_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ +// tt_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); + tt_int_op(0, OP_EQ, len2); + tt_assert(cp1 > cp2); /* Make sure we really added something. */ + + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + ZSTD_METHOD, 1, LOG_WARN)); + /* Make sure it compressed right. */ + tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); + tt_int_op(21, OP_EQ, len2); + + done: + if (state) + tor_compress_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +#else + (void)arg; + tt_assert(! tor_compress_supports_method(ZSTD_METHOD)); + + done: + ; +#endif // HAVE_ZSTD. } /** Run unit tests for mmap() wrapper functionality. */ @@ -5717,6 +5944,8 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(pow2), UTIL_LEGACY(gzip), UTIL_TEST(gzip_compression_bomb, TT_FORK), + UTIL_LEGACY(lzma), + UTIL_LEGACY(zstd), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats), diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 0e37e0d154..d3dc761c77 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -245,6 +245,7 @@ main(int c, const char **v) update_approx_time(time(NULL)); options = options_new(); tor_threads_init(); + tor_compress_init(); network_init(); diff --git a/src/trace/debug.h b/src/trace/debug.h new file mode 100644 index 0000000000..3a1652543a --- /dev/null +++ b/src/trace/debug.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TRACE_LOG_DEBUG_H +#define TOR_TRACE_LOG_DEBUG_H + +#include "torlog.h" + +/* Stringify pre-processor trick. */ +#define XSTR(d) STR(d) +#define STR(s) #s + +/* Send every event to a debug log level. This is useful to debug new trace + * events without implementing them for a specific event tracing framework. + * Note that the arguments are ignored since at this step we do not know the + * types and amount there is. */ + +/* Example on how to map a tracepoint to log_debug(). */ +#undef tor_trace +#define tor_trace(subsystem, name, args...) \ + log_debug(LD_GENERAL, "Trace event \"" XSTR(name) "\" from " \ + "\"" XSTR(subsystem) "\" hit. " \ + "(line "XSTR(__LINE__) ")") + +#endif /* TOR_TRACE_LOG_DEBUG_H */ diff --git a/src/trace/events.h b/src/trace/events.h new file mode 100644 index 0000000000..1be1fd596e --- /dev/null +++ b/src/trace/events.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file events.h + * \brief Header file for Tor event tracing. + **/ + +#ifndef TOR_TRACE_EVENTS_H +#define TOR_TRACE_EVENTS_H + +/* + * The following defines a generic event tracing function name that has to be + * used to trace events in the code base. + * + * That generic function is then defined by a event tracing framework. For + * instance, the "log debug" framework sends all trace events to log_debug() + * which is defined in src/trace/debug.h which can only be enabled at compile + * time (--enable-event-tracing-debug). + * + * By default, every trace events in the code base are replaced by a NOP. See + * doc/HACKING/Tracing.md for more information on how to use event tracing or + * add events. + */ + +#ifdef TOR_EVENT_TRACING_ENABLED +/* Map every trace event to a per subsystem macro. */ +#define tor_trace(subsystem, name, ...) \ + tor_trace_##subsystem(name, __VA_ARGS__) + +/* Enable event tracing for the debug framework where all trace events are + * mapped to a log_debug(). */ +#ifdef USE_EVENT_TRACING_DEBUG +#include "trace/debug.h" +#endif + +#else /* TOR_EVENT_TRACING_ENABLED */ + +/* Reaching this point, we NOP every event declaration because event tracing + * is not been enabled at compile time. */ +#define tor_trace(subsystem, name, args...) + +#endif /* TOR_EVENT_TRACING_ENABLED */ + +#endif /* TOR_TRACE_EVENTS_H */ diff --git a/src/trace/include.am b/src/trace/include.am new file mode 100644 index 0000000000..3285b04de6 --- /dev/null +++ b/src/trace/include.am @@ -0,0 +1,22 @@ +# Include the src/ so we can use the trace/events.h statement when including +# any file in that directory. +AM_CPPFLAGS += -I$(srcdir)/src + +noinst_LIBRARIES += \ + src/trace/libor-trace.a +LIBOR_TRACE_A_SOURCES = \ + src/trace/trace.c + +TRACEHEADERS = \ + src/trace/trace.h \ + src/trace/events.h + +if USE_EVENT_TRACING_DEBUG +TRACEHEADERS += \ + src/trace/debug.h +endif + +# Library source files. +src_trace_libor_trace_a_SOURCES = $(LIBOR_TRACE_A_SOURCES) + +noinst_HEADERS+= $(TRACEHEADERS) diff --git a/src/trace/trace.c b/src/trace/trace.c new file mode 100644 index 0000000000..fcdb80091f --- /dev/null +++ b/src/trace/trace.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "trace.h" + +/** Initialize the tracing library. */ +void +tor_trace_init(void) +{ +} + diff --git a/src/trace/trace.h b/src/trace/trace.h new file mode 100644 index 0000000000..28fcd8eea8 --- /dev/null +++ b/src/trace/trace.h @@ -0,0 +1,10 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TRACE_TRACE_H +#define TOR_TRACE_TRACE_H + +void tor_trace_init(void); + +#endif // TOR_TRACE_TRACE_H + |