From ca632144e59304ce2e41a32ddd1cf302d651cc68 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Thu, 4 May 2017 16:21:05 +0200 Subject: Use dir_compressed(_len) instead of dir_z(_len). This patch renames `dir_z` to `dir_compressed` and `dir_z_len` to `dir_compressed_len`. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 10 +++++++--- src/or/dirserv.c | 12 ++++++------ src/or/or.h | 4 ++-- 3 files changed, 15 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index da92e0f897..83a2e6c598 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3885,7 +3885,9 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compressed ? d->dir_z_len : d->dir_len); + body_len += compressed + ? d->dir_compressed_len + : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); @@ -3917,8 +3919,10 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compressed ? d->dir_z : d->dir, - compressed ? d->dir_z_len : d->dir_len, + connection_write_to_buf(compressed + ? d->dir_compressed : d->dir, + compressed + ? d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } vote_done: diff --git a/src/or/dirserv.c b/src/or/dirserv.c index c25dbf8420..bbff13fb9a 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1179,8 +1179,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len), + d->dir, d->dir_len, ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; @@ -1191,7 +1191,7 @@ static void clear_cached_dir(cached_dir_t *d) { tor_free(d->dir); - tor_free(d->dir_z); + tor_free(d->dir_compressed); memset(d, 0, sizeof(cached_dir_t)); } @@ -3508,7 +3508,7 @@ spooled_resource_estimate_size(const spooled_resource_t *spooled, if (cached == NULL) { return 0; } - size_t result = compressed ? cached->dir_z_len : cached->dir_len; + size_t result = compressed ? cached->dir_compressed_len : cached->dir_len; return result; } } @@ -3567,8 +3567,8 @@ spooled_resource_flush_some(spooled_resource_t *spooled, int64_t total_len; const char *ptr; if (cached) { - total_len = cached->dir_z_len; - ptr = cached->dir_z; + total_len = cached->dir_compressed_len; + ptr = cached->dir_compressed; } else { total_len = spooled->cce_len; ptr = (const char *)spooled->cce_body; diff --git a/src/or/or.h b/src/or/or.h index 297ec47fc1..acbf8cebbb 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1934,9 +1934,9 @@ typedef struct addr_policy_t { * compressed form. */ typedef struct cached_dir_t { char *dir; /**< Contents of this object, NUL-terminated. */ - char *dir_z; /**< Compressed contents of this object. */ + char *dir_compressed; /**< Compressed contents of this object. */ size_t dir_len; /**< Length of dir (not counting its NUL). */ - size_t dir_z_len; /**< Length of dir_z. */ + size_t dir_compressed_len; /**< Length of dir_compressed. */ time_t published; /**< When was this object published. */ common_digests_t digests; /**< Digests of this object (networkstatus only) */ /** Sha3 digest (also ns only) */ -- cgit v1.2.3-54-g00ecf From fbef257c43b90bd5d80831808e162c57ce876283 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 8 May 2017 13:52:40 +0200 Subject: Handle x-zstd and x-tor-lzma in parse_http_response(). See: https://bugs.torproject.org/21667 --- src/or/directory.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 83a2e6c598..c7c3d38e8a 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2056,6 +2056,10 @@ parse_http_response(const char *headers, int *code, time_t *date, *compression = ZLIB_METHOD; } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) { *compression = GZIP_METHOD; + } else if (!strcmp(enc, "x-zstd")) { + *compression = ZSTD_METHOD; + } else if (!strcmp(enc, "x-tor-lzma")) { + *compression = LZMA_METHOD; } else { log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", escaped(enc)); -- cgit v1.2.3-54-g00ecf From 61b6de5906b92fa1354efe60aa1506ab215d60b3 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 8 May 2017 14:00:44 +0200 Subject: Handle Zstandard and LZMA in our check for correct guessed compression. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index c7c3d38e8a..3df5be9d34 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2309,6 +2309,10 @@ connection_dir_client_reached_eof(dir_connection_t *conn) description1 = "as deflated"; else if (compression == GZIP_METHOD) description1 = "as gzipped"; + else if (compression == ZSTD_METHOD) + description1 = "as Zstandard compressed"; + else if (compression == LZMA_METHOD) + description1 = "as LZMA compressed"; else if (compression == NO_METHOD) description1 = "as uncompressed"; else @@ -2317,6 +2321,10 @@ connection_dir_client_reached_eof(dir_connection_t *conn) description2 = "deflated"; else if (guessed == GZIP_METHOD) description2 = "gzipped"; + else if (guessed == ZSTD_METHOD) + description2 = "Zstandard compressed"; + else if (guessed == LZMA_METHOD) + description2 = "LZMA compressed"; else if (!plausible) description2 = "confusing binary junk"; else -- cgit v1.2.3-54-g00ecf From 630563719728e039364b1e4cc2ada9017405a383 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 8 May 2017 16:25:20 +0200 Subject: Use tor_compress_supports_method() instead of constants. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 3df5be9d34..56d1cca333 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2336,14 +2336,14 @@ connection_dir_client_reached_eof(dir_connection_t *conn) description2, (compression>0 && guessed>0)?" Trying both.":""); } - /* Try declared compression first if we can. */ - if (compression == GZIP_METHOD || compression == ZLIB_METHOD) + /* Try declared compression first if we can. + * tor_compress_supports_method() also returns true for NO_METHOD. */ + if (tor_compress_supports_method(compression)) tor_uncompress(&new_body, &new_len, body, body_len, compression, !allow_partial, LOG_PROTOCOL_WARN); /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ - if (!new_body && - (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && + if (!new_body && tor_compress_supports_method(guessed) && compression != guessed) tor_uncompress(&new_body, &new_len, body, body_len, guessed, !allow_partial, LOG_PROTOCOL_WARN); -- cgit v1.2.3-54-g00ecf From a3a31fa120a62d51e136317df84a7202dfad214d Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Thu, 11 May 2017 01:43:37 +0200 Subject: Send "Accept-Encoding" to directory servers. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 30 ++++++++++++++++++++++++++++++ src/or/directory.h | 1 + 2 files changed, 31 insertions(+) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 56d1cca333..ca493888c2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1667,6 +1667,7 @@ directory_send_command(dir_connection_t *conn, char decorated_address[128]; smartlist_t *headers = smartlist_new(); char *url; + char *accept_encoding; size_t url_len; char request[8192]; size_t request_len, total_request_len = 0; @@ -1723,6 +1724,12 @@ directory_send_command(dir_connection_t *conn, proxystring[0] = 0; } + /* Add Accept-Encoding. */ + accept_encoding = accept_encoding_header(); + smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n", + accept_encoding); + tor_free(accept_encoding); + /* Add additional headers, if any */ { config_line_t *h; @@ -3333,6 +3340,29 @@ parse_accept_encoding_header(const char *h) return result; } +/** Return a newly allocated string containing a comma separated list of + * supported encodings. */ +STATIC char * +accept_encoding_header(void) +{ + smartlist_t *methods = smartlist_new(); + char *header = NULL; + compress_method_t method; + + // FIXME(ahf): Should we rename `srv_meth_pref_precompressed` and use this + // instead to define the order in the directory module rather than the order + // defined in the compression module? + for (method = UNKNOWN_METHOD; method > NO_METHOD; --method) { + if (tor_compress_supports_method(method)) + smartlist_add(methods, (char *)compression_method_get_name(method)); + } + + header = smartlist_join_strings(methods, ", ", 0, NULL); + smartlist_free(methods); + + return header; +} + /** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more diff --git a/src/or/directory.h b/src/or/directory.h index 125333da37..a015c7045d 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -160,6 +160,7 @@ struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, const struct get_handler_args_t *args); STATIC int directory_handle_command(dir_connection_t *conn); +STATIC char *accept_encoding_header(void); #endif -- cgit v1.2.3-54-g00ecf From cf2f7a1bea3beb03368fc309e7bf7d9ba598b6dd Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Thu, 11 May 2017 03:32:51 +0200 Subject: Decide compression method in the various handle_* functions(). See: https://bugs.torproject.org/21667 --- src/or/directory.c | 89 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index ca493888c2..78e0fb3120 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3706,6 +3706,22 @@ find_best_diff(const smartlist_t *digests, int flav, return NULL; } +/** Try to find the best supported compression method possible from a given + * compression_methods. Return NO_METHOD if no mutually supported + * compression method could be found. */ +static compress_method_t +find_best_compression_method(unsigned compression_methods) +{ + unsigned u; + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + if (compression_methods & (1u<url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); const time_t if_modified_since = args->if_modified_since; int clear_spool = 0; @@ -3786,11 +3803,11 @@ handle_get_current_consensus(dir_connection_t *conn, } else if (flavor) { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, (uint8_t*)flavor, strlen(flavor)); - compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + compression_used = compress_method; } else { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, NULL, 0); - compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + compression_used = compress_method; } tor_free(flavor); smartlist_add(conn->spool, spooled); @@ -3806,7 +3823,7 @@ handle_get_current_consensus(dir_connection_t *conn, size_t size_guess = 0; int n_expired = 0; dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, - compressed, + compress_method != NO_METHOD, &size_guess, &n_expired); @@ -3848,8 +3865,8 @@ handle_get_current_consensus(dir_connection_t *conn, write_http_response_header(conn, -1, compression_used, smartlist_len(conn->spool) == 1 ? lifetime : 0); - if (! compressed) - conn->compress_state = tor_compress_new(0, ZLIB_METHOD, + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(0, compress_method, HIGH_COMPRESSION); /* Prime the connection with some data. */ @@ -3870,7 +3887,8 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); { int current; ssize_t body_len = 0; @@ -3927,13 +3945,13 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compressed + body_len += compress_method != NO_METHOD ? d->dir_compressed_len : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); - if (compressed) { + if (compress_method != NO_METHOD) { estimated_len += ln/2; } else { body_len += ln; estimated_len += ln; @@ -3945,12 +3963,12 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } write_http_response_header(conn, body_len ? body_len : -1, - compressed ? ZLIB_METHOD : NO_METHOD, + compress_method, lifetime); if (smartlist_len(items)) { - if (compressed) { - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_compress(c, strlen(c), conn, 0)); @@ -3961,9 +3979,9 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compressed + connection_write_to_buf(compress_method != NO_METHOD ? d->dir_compressed : d->dir, - compressed + compress_method != NO_METHOD ? d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } @@ -3982,7 +4000,8 @@ static int handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); int clear_spool = 1; { conn->spool = smartlist_new(); @@ -3993,7 +4012,8 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); size_t size_guess = 0; - dirserv_spool_remove_missing_and_guess_size(conn, 0, compressed, + dirserv_spool_remove_missing_and_guess_size(conn, 0, + compress_method != NO_METHOD, &size_guess, NULL); if (smartlist_len(conn->spool) == 0) { write_http_status_line(conn, 404, "Not found"); @@ -4009,11 +4029,11 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) clear_spool = 0; write_http_response_header(conn, -1, - compressed ? ZLIB_METHOD : NO_METHOD, + compress_method, MICRODESC_CACHE_LIFETIME); - if (compressed) - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -4034,7 +4054,8 @@ static int handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); const or_options_t *options = get_options(); int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || @@ -4073,8 +4094,8 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) size_t size_guess = 0; int n_expired = 0; dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff, - compressed, &size_guess, - &n_expired); + compress_method != NO_METHOD, + &size_guess, &n_expired); /* If we are the bridge authority and the descriptor is a bridge * descriptor, remember that we served this descriptor for desc stats. */ @@ -4104,11 +4125,9 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) dir_conn_clear_spool(conn); goto done; } - write_http_response_header(conn, -1, - compressed ? ZLIB_METHOD : NO_METHOD, - cache_lifetime); - if (compressed) - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + write_http_response_header(conn, -1, compress_method, cache_lifetime); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -4129,7 +4148,8 @@ static int handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new(); @@ -4192,16 +4212,19 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) SMARTLIST_FOREACH(certs, authority_cert_t *, c, len += c->cache_info.signed_descriptor_len); - if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) { + if (global_write_bucket_low(TO_CONN(conn), + compress_method != NO_METHOD ? len/2 : len, + 2)) { write_http_status_line(conn, 503, "Directory busy, try again later"); goto keys_done; } - write_http_response_header(conn, compressed?-1:len, - compressed ? ZLIB_METHOD : NO_METHOD, + write_http_response_header(conn, + compress_method != NO_METHOD ? -1 : len, + compress_method, 60*60); - if (compressed) { - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_compress( -- cgit v1.2.3-54-g00ecf From f8218b5adaa54c61c3b79ad7f055e1a4a433daa0 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 12:21:49 +0200 Subject: Use compression_method_get_by_name() instead of explicit checks. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 78e0fb3120..bcd2eba1f9 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2057,20 +2057,15 @@ parse_http_response(const char *headers, int *code, time_t *date, if (!strcmpstart(s, "Content-Encoding: ")) { enc = s+18; break; }); - if (!enc || !strcmp(enc, "identity")) { + + if (enc == NULL) *compression = NO_METHOD; - } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) { - *compression = ZLIB_METHOD; - } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) { - *compression = GZIP_METHOD; - } else if (!strcmp(enc, "x-zstd")) { - *compression = ZSTD_METHOD; - } else if (!strcmp(enc, "x-tor-lzma")) { - *compression = LZMA_METHOD; - } else { - log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", - escaped(enc)); - *compression = UNKNOWN_METHOD; + else { + *compression = compression_method_get_by_name(enc); + + if (*compression == UNKNOWN_METHOD) + log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", + escaped(enc)); } } SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s)); -- cgit v1.2.3-54-g00ecf From 3a05687c6d4c1ed11707e53b5dbec1b81e2a9c0a Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 12:45:00 +0200 Subject: Add API for getting human readable descriptions of a compress_method_t See: https://bugs.torproject.org/21667 --- src/common/compress.c | 26 ++++++++++++++++++++++++++ src/common/compress.h | 1 + 2 files changed, 27 insertions(+) (limited to 'src') diff --git a/src/common/compress.c b/src/common/compress.c index 6fe4569868..6513029f9c 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -343,6 +343,32 @@ compression_method_get_name(compress_method_t method) return NULL; } +/** Table of compression human readable method names. */ +static const struct { + compress_method_t method; + const char *name; +} compression_method_human_names[] = { + { NO_METHOD, "uncompressed" }, + { GZIP_METHOD, "gzipped" }, + { ZLIB_METHOD, "deflated" }, + { LZMA_METHOD, "LZMA compressed" }, + { ZSTD_METHOD, "Zstandard compressed" }, + { UNKNOWN_METHOD, "unknown encoding" }, +}; + +/** Return a human readable string representation of the compression method + * method, or NULL if the method isn't recognized. */ +const char * +compression_method_get_human_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) { + if (method == compression_method_human_names[i].method) + return compression_method_human_names[i].name; + } + return NULL; +} + /** Return the compression method represented by the string name, or * UNKNOWN_METHOD if the string isn't recognized. */ compress_method_t diff --git a/src/common/compress.h b/src/common/compress.h index 5b47c5d458..7c0dc14061 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -50,6 +50,7 @@ int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); int tor_compress_supports_method(compress_method_t method); unsigned tor_compress_get_supported_method_bitmask(void); const char * compression_method_get_name(compress_method_t method); +const char *compression_method_get_human_name(compress_method_t method); compress_method_t compression_method_get_by_name(const char *name); const char *tor_compress_version_str(compress_method_t method); -- cgit v1.2.3-54-g00ecf From ef187bc280f1501bd7918b9c76a692027b320a67 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 12:54:16 +0200 Subject: Use compression_method_get_human_name() in connection_dir_client_reached_eof() This patch refactors connection_dir_client_reached_eof() to use compression_method_get_human_name() to set description1 and description2 variables. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index bcd2eba1f9..ed4f961acf 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2307,32 +2307,18 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (compression == UNKNOWN_METHOD || guessed != compression) { /* Tell the user if we don't believe what we're told about compression.*/ const char *description1, *description2; - if (compression == ZLIB_METHOD) - description1 = "as deflated"; - else if (compression == GZIP_METHOD) - description1 = "as gzipped"; - else if (compression == ZSTD_METHOD) - description1 = "as Zstandard compressed"; - else if (compression == LZMA_METHOD) - description1 = "as LZMA compressed"; - else if (compression == NO_METHOD) - description1 = "as uncompressed"; - else - description1 = "with an unknown Content-Encoding"; - if (guessed == ZLIB_METHOD) - description2 = "deflated"; - else if (guessed == GZIP_METHOD) - description2 = "gzipped"; - else if (guessed == ZSTD_METHOD) - description2 = "Zstandard compressed"; - else if (guessed == LZMA_METHOD) - description2 = "LZMA compressed"; - else if (!plausible) + + description1 = compression_method_get_human_name(compression); + + if (BUG(description1 == NULL)) + description1 = compression_method_get_human_name(UNKNOWN_METHOD); + + if (guessed == UNKNOWN_METHOD && !plausible) description2 = "confusing binary junk"; else - description2 = "uncompressed"; + description2 = compression_method_get_human_name(guessed); - log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, " + log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled as %s, " "but it seems to be %s.%s", conn->base_.address, conn->base_.port, description1, description2, -- cgit v1.2.3-54-g00ecf From 59d17ca2bb6ab243bd58b29d26ccf6dd10741abe Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 13:17:43 +0200 Subject: Fix indentation when using the ternary operator in handle_get_status_vote(). See: https://bugs.torproject.org/21667 --- src/or/directory.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index ed4f961acf..7965ffd0ac 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3926,9 +3926,8 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compress_method != NO_METHOD - ? d->dir_compressed_len - : d->dir_len); + body_len += compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); @@ -3960,10 +3959,10 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compress_method != NO_METHOD - ? d->dir_compressed : d->dir, - compress_method != NO_METHOD - ? d->dir_compressed_len : d->dir_len, + connection_write_to_buf(compress_method != NO_METHOD ? + d->dir_compressed : d->dir, + compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } vote_done: -- cgit v1.2.3-54-g00ecf From 141f6e321190b7c8f928aba88ad0be8f553ee282 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 13:37:29 +0200 Subject: Add client_meth_pref array to define client compression preference. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 7965ffd0ac..d9d402afe2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3321,6 +3321,16 @@ parse_accept_encoding_header(const char *h) return result; } +/** Array of compression methods to use (if supported) for requesting + * compressed data, ordered from best to worst. */ +static compress_method_t client_meth_pref[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + /** Return a newly allocated string containing a comma separated list of * supported encodings. */ STATIC char * @@ -3329,11 +3339,10 @@ accept_encoding_header(void) smartlist_t *methods = smartlist_new(); char *header = NULL; compress_method_t method; + unsigned i; - // FIXME(ahf): Should we rename `srv_meth_pref_precompressed` and use this - // instead to define the order in the directory module rather than the order - // defined in the compression module? - for (method = UNKNOWN_METHOD; method > NO_METHOD; --method) { + for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { + method = client_meth_pref[i]; if (tor_compress_supports_method(method)) smartlist_add(methods, (char *)compression_method_get_name(method)); } -- cgit v1.2.3-54-g00ecf From 7a3efe25d9625a566d08eae7ab9ffd862c1d4a6d Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 13:55:18 +0200 Subject: Use different preferences for compression methods when streaming. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index d9d402afe2..4a528f23ae 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3296,6 +3296,15 @@ static compress_method_t srv_meth_pref_precompressed[] = { NO_METHOD }; +/** Array of compression methods to use (if supported) for serving + * streamed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_streaming_compression[] = { + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + /** Parse the compression methods listed in an Accept-Encoding header h, * and convert them to a bitfield where compression method x is supported if * and only if 1 << x is set in the bitfield. */ @@ -3700,11 +3709,22 @@ find_best_diff(const smartlist_t *digests, int flav, * compression_methods. Return NO_METHOD if no mutually supported * compression method could be found. */ static compress_method_t -find_best_compression_method(unsigned compression_methods) +find_best_compression_method(unsigned compression_methods, int stream) { unsigned u; - for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { - compress_method_t method = srv_meth_pref_precompressed[u]; + compress_method_t *methods; + size_t length; + + if (stream) { + methods = srv_meth_pref_streaming_compression; + length = ARRAY_LENGTH(srv_meth_pref_streaming_compression); + } else { + methods = srv_meth_pref_precompressed; + length = ARRAY_LENGTH(srv_meth_pref_precompressed); + } + + for (u = 0; u < length; ++u) { + compress_method_t method = methods[u]; if (compression_methods & (1u<url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 0); const time_t if_modified_since = args->if_modified_since; int clear_spool = 0; @@ -3878,7 +3898,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); { int current; ssize_t body_len = 0; @@ -3990,7 +4010,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); int clear_spool = 1; { conn->spool = smartlist_new(); @@ -4044,7 +4064,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); const or_options_t *options = get_options(); int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || @@ -4138,7 +4158,7 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new(); -- cgit v1.2.3-54-g00ecf From 6da31ec4848b23db4b923ff5141c24b82f466c8d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 May 2017 10:48:02 -0400 Subject: consdiffmgr: Extract the code for compressing and storing We're going to use this for consensuses too. --- src/or/consdiffmgr.c | 142 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 43 deletions(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index d478101c6b..b108c8028e 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -1071,6 +1071,91 @@ typedef struct compressed_result_t { size_t bodylen; } compressed_result_t; +/** + * Compress the bytestring input of length len using the + * n_methods compression methods listed in the array methods. + * + * For each successful compression, set the fields in the results_out + * array in the position corresponding to the compression method. Use + * labels_in as a basis for the labels of the result. + * + * Return 0 if all compression succeeded; -1 if any failed. + */ +static int +compress_multiple(compressed_result_t *results_out, int n_methods, + const compress_method_t *methods, + const uint8_t *input, size_t len, + const config_line_t *labels_in) +{ + int rv = 0; + int i; + for (i = 0; i < n_methods; ++i) { + compress_method_t method = methods[i]; + const char *methodname = compression_method_get_name(method); + char *result; + size_t sz; + if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) { + results_out[i].body = (uint8_t*)result; + results_out[i].bodylen = sz; + results_out[i].labels = config_lines_dup(labels_in); + cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST, + results_out[i].body, + results_out[i].bodylen); + config_line_prepend(&results_out[i].labels, + LABEL_COMPRESSION_TYPE, + methodname); + } else { + rv = -1; + } + } + return rv; +} + +/** + * Given an array of n compressed_result_t in results, + * as produced by compress_multiple, store them all into the + * consdiffmgr, and store handles to them in the handles_out + * array. + * + * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none + * was stored. + */ +static cdm_diff_status_t +store_multiple(consensus_cache_entry_handle_t **handles_out, + int n, + const compress_method_t *methods, + const compressed_result_t *results, + const char *description) +{ + cdm_diff_status_t status = CDM_DIFF_ERROR; + consdiffmgr_ensure_space_for_files(n); + + int i; + for (i = 0; i < n; ++i) { + compress_method_t method = methods[i]; + uint8_t *body_out = results[i].body; + size_t bodylen_out = results[i].bodylen; + config_line_t *labels = results[i].labels; + const char *methodname = compression_method_get_name(method); + if (body_out && bodylen_out && labels) { + /* Success! Store the results */ + log_info(LD_DIRSERV, "Adding %s, compressed with %s", + description, methodname); + + consensus_cache_entry_t *ent = + consensus_cache_add(cdm_cache_get(), + labels, + body_out, + bodylen_out); + + status = CDM_DIFF_PRESENT; + handles_out[i] = consensus_cache_entry_handle_new(ent); + consensus_cache_entry_decref(ent); + } + } + return status; +} + /** * An object passed to a worker thread that will try to produce a consensus * diff. @@ -1235,24 +1320,10 @@ consensus_diff_worker_threadfn(void *state_, void *work_) job->out[0].body, job->out[0].bodylen); - unsigned u; - for (u = 1; u < n_diff_compression_methods(); ++u) { - compress_method_t method = compress_diffs_with[u]; - const char *methodname = compression_method_get_name(method); - char *result; - size_t sz; - if (0 == tor_compress(&result, &sz, consensus_diff, difflen, method)) { - job->out[u].body = (uint8_t*)result; - job->out[u].bodylen = sz; - job->out[u].labels = config_lines_dup(common_labels); - cdm_labels_prepend_sha3(&job->out[u].labels, LABEL_SHA3_DIGEST, - job->out[u].body, - job->out[u].bodylen); - config_line_prepend(&job->out[u].labels, - LABEL_COMPRESSION_TYPE, - methodname); - } - } + compress_multiple(job->out+1, + n_diff_compression_methods()-1, + compress_diffs_with+1, + (const uint8_t*)consensus_diff, difflen, common_labels); config_free_lines(common_labels); return WQ_RPL_REPLY; @@ -1318,36 +1389,20 @@ consensus_diff_worker_replyfn(void *work_) cache = 0; } - int status = CDM_DIFF_ERROR; consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)]; memset(handles, 0, sizeof(handles)); - consdiffmgr_ensure_space_for_files(n_diff_compression_methods()); + char description[128]; + tor_snprintf(description, sizeof(description), + "consensus diff from %s to %s", + lv_from_digest, lv_to_digest); - unsigned u; - for (u = 0; u < n_diff_compression_methods(); ++u) { - compress_method_t method = compress_diffs_with[u]; - uint8_t *body_out = job->out[u].body; - size_t bodylen_out = job->out[u].bodylen; - config_line_t *labels = job->out[u].labels; - const char *methodname = compression_method_get_name(method); - if (body_out && bodylen_out && labels) { - /* Success! Store the results */ - log_info(LD_DIRSERV, "Adding consensus diff from %s to %s, " - "compressed with %s", - lv_from_digest, lv_to_digest, methodname); + int status = store_multiple(handles, + n_diff_compression_methods(), + compress_diffs_with, + job->out, + description); - consensus_cache_entry_t *ent = - consensus_cache_add(cdm_cache_get(), - labels, - body_out, - bodylen_out); - - status = CDM_DIFF_PRESENT; - handles[u] = consensus_cache_entry_handle_new(ent); - consensus_cache_entry_decref(ent); - } - } if (status != CDM_DIFF_PRESENT) { /* Failure! Nothing to do but complain */ log_warn(LD_DIRSERV, @@ -1357,6 +1412,7 @@ consensus_diff_worker_replyfn(void *work_) status = CDM_DIFF_ERROR; } + unsigned u; for (u = 0; u < ARRAY_LENGTH(handles); ++u) { compress_method_t method = compress_diffs_with[u]; if (cache) { -- cgit v1.2.3-54-g00ecf From 151cd121a2733506c69162330b210c3c45044dfa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 May 2017 11:17:59 -0400 Subject: consdiffmgr: compress incoming consensuses in the background Also, compress them in several ways. This breaks the unit tests; subsequent commits will make them pass again. --- src/or/consdiffmgr.c | 233 +++++++++++++++++++++++++++++++++++++-------------- src/or/consdiffmgr.h | 1 + 2 files changed, 173 insertions(+), 61 deletions(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index b108c8028e..2f0a653a52 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -96,6 +96,24 @@ n_diff_compression_methods(void) return ARRAY_LENGTH(compress_diffs_with); } +/** Which methods do we use for precompressing consensuses? */ +static const compress_method_t compress_consensus_with[] = { + ZLIB_METHOD, +#ifdef HAVE_LZMA + LZMA_METHOD, +#endif +#ifdef HAVE_ZSTD + ZSTD_METHOD, +#endif +}; + +/** How many different methods will we try to use for diff compression? */ +static unsigned +n_consensus_compression_methods(void) +{ + return ARRAY_LENGTH(compress_consensus_with); +} + /** Hashtable node used to remember the current status of the diff * from a given sha3 digest to the current consensus. */ typedef struct cdm_diff_t { @@ -135,13 +153,13 @@ static consdiff_cfg_t consdiff_cfg = { }; static int consdiffmgr_ensure_space_for_files(int n); +static int consensus_queue_compression_work(const char *consensus, + consensus_flavor_t flavor, + time_t valid_after); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); static void consdiffmgr_set_cache_flags(void); -/* Just gzip consensuses for now. */ -#define COMPRESS_CONSENSUS_WITH GZIP_METHOD - /* ===== * Hashtable setup * ===== */ @@ -410,11 +428,6 @@ cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after) consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); consensus_cache_entry_t *result = NULL; - if (smartlist_len(matches) > 1) { - log_warn(LD_BUG, "How odd; there appear to be two matching consensuses " - "with flavor %s published at %s.", - flavname, formatted_time); - } if (smartlist_len(matches)) { result = smartlist_get(matches, 0); } @@ -458,59 +471,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - consdiffmgr_ensure_space_for_files(1); - - { - size_t bodylen = strlen(consensus); - config_line_t *labels = NULL; - char formatted_time[ISO_TIME_LEN+1]; - format_iso_time_nospace(formatted_time, valid_after); - const char *flavname = networkstatus_get_flavor_name(flavor); - - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, - (const uint8_t *)consensus, bodylen); - { - const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { - start = consensus; - end = consensus+bodylen; - } - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, - (const uint8_t *)start, - end - start); - } - - char *body_compressed = NULL; - size_t size_compressed = 0; - if (tor_compress(&body_compressed, &size_compressed, - consensus, bodylen, COMPRESS_CONSENSUS_WITH) < 0) { - config_free_lines(labels); - return -1; - } - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST, - (const uint8_t *)body_compressed, size_compressed); - config_line_prepend(&labels, LABEL_COMPRESSION_TYPE, - compression_method_get_name(COMPRESS_CONSENSUS_WITH)); - config_line_prepend(&labels, LABEL_FLAVOR, flavname); - config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); - config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); - - entry = consensus_cache_add(cdm_cache_get(), - labels, - (const uint8_t *)body_compressed, - size_compressed); - tor_free(body_compressed); - config_free_lines(labels); - } - - if (entry) { - consensus_cache_entry_mark_for_aggressive_release(entry); - consensus_cache_entry_decref(entry); - } - - cdm_cache_dirty = 1; - return entry ? 0 : -1; + return consensus_queue_compression_work(consensus, flavor, valid_after); } /** @@ -835,6 +796,10 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) if (strmap_get(have_diff_from, va) != NULL) continue; /* we already have this one. */ smartlist_add(compute_diffs_from, ent); + /* Since we are not going to serve this as the most recent consensus + * any more, we should stop keeping it mmap'd when it's not in use. + */ + consensus_cache_entry_mark_for_aggressive_release(ent); } SMARTLIST_FOREACH_END(ent); log_info(LD_DIRSERV, @@ -1147,6 +1112,8 @@ store_multiple(consensus_cache_entry_handle_t **handles_out, labels, body_out, bodylen_out); + if (BUG(ent == NULL)) + continue; status = CDM_DIFF_PRESENT; handles_out[i] = consensus_cache_entry_handle_new(ent); @@ -1464,3 +1431,147 @@ consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, return -1; } +/** + * Holds requests and replies for consensus_compress_workers. + */ +typedef struct consensus_compress_worker_job_t { + char *consensus; + size_t consensus_len; + consensus_flavor_t flavor; + time_t valid_after; + compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)]; +} consensus_compress_worker_job_t; + +/** + * Free all resources held in job + */ +static void +consensus_compress_worker_job_free(consensus_compress_worker_job_t *job) +{ + if (!job) + return; + tor_free(job->consensus); + unsigned u; + for (u = 0; u < n_consensus_compression_methods(); ++u) { + config_free_lines(job->out[u].labels); + tor_free(job->out[u].body); + } + tor_free(job); +} +/** + * Worker function. This function runs inside a worker thread and receives + * a consensus_compress_worker_job_t as its input. + */ +static workqueue_reply_t +consensus_compress_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + consensus_compress_worker_job_t *job = work_; + consensus_flavor_t flavor = job->flavor; + const char *consensus = job->consensus; + size_t bodylen = job->consensus_len; + time_t valid_after = job->valid_after; + + config_line_t *labels = NULL; + char formatted_time[ISO_TIME_LEN+1]; + format_iso_time_nospace(formatted_time, valid_after); + + const char *flavname = networkstatus_get_flavor_name(flavor); + + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, + (const uint8_t *)consensus, bodylen); + { + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(consensus, + &start, &end) < 0) { + start = consensus; + end = consensus+bodylen; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, + (const uint8_t *)start, + end - start); + } + config_line_prepend(&labels, LABEL_FLAVOR, flavname); + config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); + config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + compress_multiple(job->out, + n_consensus_compression_methods(), + compress_consensus_with, + (const uint8_t*)consensus, bodylen, labels); + config_free_lines(labels); + return WQ_RPL_REPLY; +} + +/** + * Worker function: This function runs in the main thread, and receives + * a consensus_diff_compress_job_t that the worker thread has already + * processed. + */ +static void +consensus_compress_worker_replyfn(void *work_) +{ + consensus_compress_worker_job_t *job = work_; + + consensus_cache_entry_handle_t *handles[ + ARRAY_LENGTH(compress_consensus_with)]; + memset(handles, 0, sizeof(handles)); + + store_multiple(handles, + n_consensus_compression_methods(), + compress_consensus_with, + job->out, + "consensus"); + cdm_cache_dirty = 1; + + consensus_compress_worker_job_free(job); +} + +/** + * If true, we compress in worker threads. + */ +static int background_compression = 0; + +/** + * Queue a job to compress consensus and store its compressed + * text in the cache. + */ +static int +consensus_queue_compression_work(const char *consensus, + consensus_flavor_t flavor, + time_t valid_after) +{ + consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->consensus = tor_strdup(consensus); + job->consensus_len = strlen(consensus); + job->flavor = flavor; + job->valid_after = valid_after; + + if (background_compression) { + workqueue_entry_t *work; + work = cpuworker_queue_work(consensus_compress_worker_threadfn, + consensus_compress_worker_replyfn, + job); + if (!work) { + consensus_compress_worker_job_free(job); + return -1; + } + + return 0; + } else { + consensus_compress_worker_threadfn(NULL, job); + consensus_compress_worker_replyfn(job); + return 0; + } +} + +/** + * Tell the consdiffmgr backend to compress consensuses in worker threads. + */ +void +consdiffmgr_enable_background_compression(void) +{ + // This isn't the default behavior because it would break unit tests. + background_compression = 1; +} + diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index 048dae432c..cbff599381 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -32,6 +32,7 @@ consdiff_status_t consdiffmgr_find_diff_from( compress_method_t method); void consdiffmgr_rescan(void); int consdiffmgr_cleanup(void); +void consdiffmgr_enable_background_compression(void); void consdiffmgr_configure(const consdiff_cfg_t *cfg); struct sandbox_cfg_elem; int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg); -- cgit v1.2.3-54-g00ecf From 7b0dcf5c4a081ef892d79a9a3e36a6d4d1ec31f9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 May 2017 11:32:35 -0400 Subject: Cleanup logic: only retain zlib-compressed consensuses Now that we're making a bunch of these with consdiffmgr, we should throw out all but one when we get a newer consensus. --- src/or/consdiffmgr.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 2f0a653a52..4fccb430db 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -114,6 +114,12 @@ n_consensus_compression_methods(void) return ARRAY_LENGTH(compress_consensus_with); } +/** For which compression method do we retain old consensuses? There's no + * need to keep all of them, since we won't be serving them. We'll + * go with ZLIB_METHOD because it's pretty fast and everyone has it. + */ +#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD + /** Hashtable node used to remember the current status of the diff * from a given sha3 digest to the current consensus. */ typedef struct cdm_diff_t { @@ -645,6 +651,43 @@ consdiffmgr_cleanup(void) smartlist_clear(diffs); } + // 3. Delete all consensuses except the most recent that are compressed with + // an un-preferred method. + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + /* Determine the most recent consensus of this flavor */ + consensus_cache_find_all(consensuses, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname); + consensus_cache_entry_t *most_recent = + sort_and_find_most_recent(consensuses); + if (most_recent == NULL) + continue; + const char *most_recent_sha3_uncompressed = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + const char *retain_methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + + + if (BUG(most_recent_sha3_uncompressed == NULL)) + continue; + SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) { + const char *lv_sha3_uncompressed = + consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(! lv_sha3_uncompressed)) + continue; + if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed)) + continue; // This _is_ the most recent. + const char *lv_methodname = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) { + consensus_cache_entry_mark_for_removal(ent); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(ent); + } + smartlist_free(objects); smartlist_free(consensuses); smartlist_free(diffs); @@ -748,10 +791,14 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) // 1. find the most recent consensus, and the ones that we might want // to diff to it. + const char *methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + matches = smartlist_new(); consensus_cache_find_all(matches, cdm_cache_get(), LABEL_FLAVOR, flavname); consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname); consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); if (!most_recent) { log_info(LD_DIRSERV, "No 'most recent' %s consensus found; " -- cgit v1.2.3-54-g00ecf From 8100305e71fb1072831de1666cbb39196d1ae162 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 May 2017 11:47:45 -0400 Subject: consdiffmgr: expose cached consensuses --- src/or/consdiffmgr.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/or/consdiffmgr.h | 5 +++++ 2 files changed, 64 insertions(+) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 4fccb430db..0feb84ac38 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -120,6 +120,12 @@ n_consensus_compression_methods(void) */ #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD +/** Handles pointing to the latest consensus entries as compressed and + * stored. */ +static consensus_cache_entry_handle_t * + latest_consensus[N_CONSENSUS_FLAVORS] + [ARRAY_LENGTH(compress_consensus_with)]; + /** Hashtable node used to remember the current status of the diff * from a given sha3 digest to the current consensus. */ typedef struct cdm_diff_t { @@ -510,6 +516,41 @@ sort_and_find_most_recent(smartlist_t *lst) } } +/** + * If we know a consensus with the flavor flavor compressed with + * method, set *entry_out to that value. Return values are as + * for consdiffmgr_find_diff_from(). + */ +consdiff_status_t +consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method) +{ + int pos=-1; + unsigned i; + tor_assert(flavor < N_CONSENSUS_FLAVORS); + + // Find the index of method withing compress_consensus_with + for (i = 0; i < n_consensus_compression_methods(); ++i) { + if (compress_consensus_with[i] == method) { + pos = i; + break; + } + } + if (pos < 0) { + // We don't compress consensuses with this method. + return CONSDIFF_NOT_FOUND; + } + consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos]; + if (!handle) + return CONSDIFF_NOT_FOUND; + *entry_out = consensus_cache_entry_handle_get(handle); + if (entry_out) + return CONSDIFF_AVAILABLE; + else + return CONSDIFF_NOT_FOUND; +} + /** * Look up consensus_cache_entry_t for the consensus of type flavor, * from the source consensus with the specified digest (which must be SHA3). @@ -1063,6 +1104,14 @@ consdiffmgr_free_all(void) next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); cdm_diff_free(this); } + int i; + unsigned j; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + for (j = 0; j < n_consensus_compression_methods(); ++j) { + consensus_cache_entry_handle_free(latest_consensus[i][j]); + } + } + memset(latest_consensus, 0, sizeof(latest_consensus)); consensus_cache_free(cons_diff_cache); cons_diff_cache = NULL; } @@ -1571,6 +1620,16 @@ consensus_compress_worker_replyfn(void *work_) "consensus"); cdm_cache_dirty = 1; + unsigned u; + consensus_flavor_t f = job->flavor; + tor_assert(f < N_CONSENSUS_FLAVORS); + for (u = 0; u < ARRAY_LENGTH(handles); ++u) { + if (handles[u] == NULL) + continue; + consensus_cache_entry_handle_free(latest_consensus[f][u]); + latest_consensus[f][u] = handles[u]; + } + consensus_compress_worker_job_free(job); } diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index cbff599381..5f7e944f55 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -23,6 +23,11 @@ struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, const networkstatus_t *as_parsed); +consdiff_status_t consdiffmgr_find_consensus( + struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method); + consdiff_status_t consdiffmgr_find_diff_from( struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, -- cgit v1.2.3-54-g00ecf From 30dfb36148c05b8757f3ac263a8473b7aeba0ac5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 May 2017 11:58:19 -0400 Subject: consdiffmgr: Reload latest consensus entries on start. --- src/or/consdiffmgr.c | 67 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 0feb84ac38..ad63ce9d52 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -516,6 +516,20 @@ sort_and_find_most_recent(smartlist_t *lst) } } +/** Return i such that compress_consensus_with[i] == method. Return + * -1 if no such i exists. */ +static int +consensus_compression_method_pos(compress_method_t method) +{ + unsigned i; + for (i = 0; i < n_consensus_compression_methods(); ++i) { + if (compress_consensus_with[i] == method) { + return i; + } + } + return -1; +} + /** * If we know a consensus with the flavor flavor compressed with * method, set *entry_out to that value. Return values are as @@ -526,17 +540,9 @@ consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, compress_method_t method) { - int pos=-1; - unsigned i; tor_assert(flavor < N_CONSENSUS_FLAVORS); - // Find the index of method withing compress_consensus_with - for (i = 0; i < n_consensus_compression_methods(); ++i) { - if (compress_consensus_with[i] == method) { - pos = i; - break; - } - } + int pos = consensus_compression_method_pos(method); if (pos < 0) { // We don't compress consensuses with this method. return CONSDIFF_NOT_FOUND; @@ -932,6 +938,48 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) strmap_free(have_diff_from, NULL); } +/** + * Scan the cache for the latest consensuses and add their handles to + * latest_consensus + */ +static void +consdiffmgr_consensus_load(void) +{ + smartlist_t *matches = smartlist_new(); + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + smartlist_clear(matches); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); + if (! most_recent) + continue; // no consensuses. + const char *most_recent_sha3 = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(most_recent_sha3 == NULL)) + continue; // LCOV_EXCL_LINE + consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED, + most_recent_sha3); + + // Everything that remains matches the most recent consensus of this + // flavor. + SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) { + const char *lv_compression = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + compress_method_t method = + compression_method_get_by_name(lv_compression); + int pos = consensus_compression_method_pos(method); + if (pos < 0) + continue; + consensus_cache_entry_handle_free(latest_consensus[flav][pos]); + latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent); + } SMARTLIST_FOREACH_END(ent); + } + smartlist_free(matches); +} + /** * Scan the cache for diffs, and add them to the hashtable. */ @@ -989,6 +1037,7 @@ consdiffmgr_rescan(void) if (cdm_cache_loaded == 0) { consdiffmgr_diffs_load(); + consdiffmgr_consensus_load(); cdm_cache_loaded = 1; } -- cgit v1.2.3-54-g00ecf From db370bb8a8fd65a902d3abd996ace019db4c296c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 May 2017 12:10:45 -0400 Subject: Test fix: expect old consensuses to be deleted if not deflate-compressed --- src/or/consdiffmgr.c | 2 +- src/or/consdiffmgr.h | 1 + src/test/test_consdiffmgr.c | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index ad63ce9d52..3e1edf0fd6 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -108,7 +108,7 @@ static const compress_method_t compress_consensus_with[] = { }; /** How many different methods will we try to use for diff compression? */ -static unsigned +STATIC unsigned n_consensus_compression_methods(void) { return ARRAY_LENGTH(compress_consensus_with); diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index 5f7e944f55..abe1ea9dc3 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -46,6 +46,7 @@ int consdiffmgr_validate(void); #ifdef CONSDIFFMGR_PRIVATE STATIC unsigned n_diff_compression_methods(void); +STATIC unsigned n_consensus_compression_methods(void); STATIC consensus_cache_t *cdm_cache_get(void); STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( consensus_flavor_t flavor, time_t valid_after); diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 9d0c5b5b71..2e5dd59292 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -759,9 +759,11 @@ test_consdiffmgr_cleanup_old_diffs(void *arg) consensus_cache_entry_incref(hold_ent); // incref, so it is preserved. /* Now add an even-more-recent consensus; this should make all previous - * diffs deletable */ + * diffs deletable, and make delete */ tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3])); - tt_int_op(2 * n_diff_compression_methods(), OP_EQ, consdiffmgr_cleanup()); + tt_int_op(2 * n_diff_compression_methods() + + (n_consensus_compression_methods() - 1) , OP_EQ, + consdiffmgr_cleanup()); tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0])); -- cgit v1.2.3-54-g00ecf From 077d3085ec30411daa6b3985ba7b10fd8ef861fa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 May 2017 12:15:24 -0400 Subject: actually enable background compresion for consensuses --- src/or/main.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/or/main.c b/src/or/main.c index f1a8cfb96e..9f0c29cf0b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2482,6 +2482,7 @@ do_main_loop(void) /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } + consdiffmgr_enable_background_compression(); /* Setup shared random protocol subsystem. */ if (authdir_mode_v3(get_options())) { -- cgit v1.2.3-54-g00ecf From a1e8ef007634c4916e1bf3f2473182aa179187d9 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 17:57:11 +0200 Subject: Fix DoubleNL warning from `make check-spaces`. --- src/or/consdiffmgr.c | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 3e1edf0fd6..b8a52dc322 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -716,7 +716,6 @@ consdiffmgr_cleanup(void) const char *retain_methodname = compression_method_get_name( RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); - if (BUG(most_recent_sha3_uncompressed == NULL)) continue; SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) { -- cgit v1.2.3-54-g00ecf From 64116ab97f14592ed67899512e8e3279df898adc Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 17:59:29 +0200 Subject: Fix tautological constant out-of-range comparison warnings. --- src/or/consdiffmgr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index b8a52dc322..a2e269323f 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -540,7 +540,7 @@ consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, compress_method_t method) { - tor_assert(flavor < N_CONSENSUS_FLAVORS); + tor_assert((int)flavor < N_CONSENSUS_FLAVORS); int pos = consensus_compression_method_pos(method); if (pos < 0) { @@ -1670,7 +1670,7 @@ consensus_compress_worker_replyfn(void *work_) unsigned u; consensus_flavor_t f = job->flavor; - tor_assert(f < N_CONSENSUS_FLAVORS); + tor_assert((int)f < N_CONSENSUS_FLAVORS); for (u = 0; u < ARRAY_LENGTH(handles); ++u) { if (handles[u] == NULL) continue; -- cgit v1.2.3-54-g00ecf From 363f4b8db5f9f642f848f2e50ae0c6bb0f1ff25e Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Sat, 13 May 2017 01:03:55 +0200 Subject: Add stub functions for querying metadata about the consensus. --- src/or/conscache.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/or/conscache.h | 8 ++++++++ 2 files changed, 67 insertions(+) (limited to 'src') diff --git a/src/or/conscache.c b/src/or/conscache.c index 5ffa129bbe..fd46cbe65e 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -374,6 +374,65 @@ consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, return 0; } +/** Read the lifetime of cached object ent into lifetime. */ +int consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, + long *lifetime) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + tor_assert(lifetime); + + // FIXME(ahf): Fill out. + *lifetime = 0; + + return 0; +} + +/** Return non-zero if the cache object found in ent is + * reasonably live, otherwise return 0. Use now to pass the + * timestamp used for comparison. */ +int consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, + time_t now) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + // FIXME(ahf): Fill out. + (void)now; + + return 1; +} + +/** Read the set of voters from the cached object ent into out. */ +int consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, + smartlist_t *out) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + // FIXME(ahf): Fill out. + (void)out; + + return 0; +} + +/** Read the valid until timestamp from the cached object ent + * into out. */ +int consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, + time_t *out) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + tor_assert(out); + + // FIXME(ahf): Fill out. + *out = time(NULL); + + return 0; +} + /** * Unmap every mmap'd element of cache that has been unused * since cutoff. diff --git a/src/or/conscache.h b/src/or/conscache.h index aef54201f0..fcc3f3e330 100644 --- a/src/or/conscache.h +++ b/src/or/conscache.h @@ -52,6 +52,14 @@ void consensus_cache_entry_mark_for_aggressive_release( int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, const uint8_t **body_out, size_t *sz_out); +int consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, + long *lifetime); +int consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, + time_t now); +int consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, + smartlist_t *out); +int consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, + time_t *out); #ifdef TOR_UNIT_TESTS int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent); -- cgit v1.2.3-54-g00ecf From ef2a62b2ff0c8db17e146a79034c8e43ee528c86 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Fri, 12 May 2017 21:17:09 +0200 Subject: Fetch the current consensus from the conscache subsystem. This patch changes handle_get_current_consensus() to make it read the current consensus document from the consensus caching subsystem. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 121 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 4a528f23ae..946792b1b3 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -13,6 +13,7 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "conscache.h" #include "consdiff.h" #include "consdiffmgr.h" #include "control.h" @@ -117,7 +118,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int was_descriptor_digests); static void dir_microdesc_download_failed(smartlist_t *failed, int status_code); -static int client_likes_consensus(networkstatus_t *v, const char *want_url); +static int client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url); static void connection_dir_close_consensus_fetches( dir_connection_t *except_this_one, const char *resource); @@ -3377,14 +3379,20 @@ accept_encoding_header(void) * consensus, 0 otherwise. */ int -client_likes_consensus(networkstatus_t *v, const char *want_url) +client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url) { smartlist_t *want_authorities = smartlist_new(); + smartlist_t *voters = smartlist_new(); int need_at_least; int have = 0; dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; + + if (consensus_cache_entry_get_voters(ent, voters) != 0) + goto done; + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; @@ -3398,7 +3406,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) continue; }; - SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + SMARTLIST_FOREACH_BEGIN(voters, networkstatus_voter_info_t *, vi) { if (smartlist_len(vi->sigs) && tor_memeq(vi->identity_digest, want_digest, want_len)) { have++; @@ -3411,8 +3419,11 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) break; } SMARTLIST_FOREACH_END(d); + done: SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); + SMARTLIST_FOREACH(voters, networkstatus_voter_info_t *, v, tor_free(v)); + smartlist_free(voters); return (have >= need_at_least); } @@ -3619,20 +3630,25 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) return 0; } -/** Warn that the consensus v of type flavor is too old and will - * not be served to clients. Rate-limit the warning to avoid logging an entry - * on every request. +/** Warn that the cached consensus cached_consensus of type + * flavor is too old and will not be served to clients. Rate-limit the + * warning to avoid logging an entry on every request. */ static void -warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) +warn_consensus_is_too_old(const struct consensus_cache_entry_t *cached_consensus, + const char *flavor, time_t now) { #define TOO_OLD_WARNING_INTERVAL (60*60) static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); char timestamp[ISO_TIME_LEN+1]; + time_t valid_until; char *dupes; + if (consensus_cache_entry_valid_until(cached_consensus, &valid_until)) + return; + if ((dupes = rate_limit_log(&warned, now))) { - format_local_iso_time(timestamp, v->valid_until); + format_local_iso_time(timestamp, valid_until); log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " "serve it to clients. It was valid until %s local time and we " "continued to serve it for up to 24 hours after it expired.%s", @@ -3705,6 +3721,35 @@ find_best_diff(const smartlist_t *digests, int flav, return NULL; } +/** Lookup the cached consensus document by the flavor found in flav. + * The prefered set of compression methods should be listed in the + * compression_methods bitfield. The compression method chosen (if any) + * is stored in compression_used_out. */ +static struct consensus_cache_entry_t * +find_best_consensus(int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + unsigned u; + + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + + if (0 == (compression_methods & (1u<compression_methods. Return NO_METHOD if no mutually supported * compression method could be found. */ @@ -3747,7 +3792,6 @@ handle_get_current_consensus(dir_connection_t *conn, /* v3 network status fetch. */ long lifetime = NETWORKSTATUS_CACHE_LIFETIME; - networkstatus_t *v; time_t now = time(NULL); const char *want_fps = NULL; char *flavor = NULL; @@ -3773,18 +3817,33 @@ handle_get_current_consensus(dir_connection_t *conn, want_fps = url+strlen(CONSENSUS_URL_PREFIX); } - v = networkstatus_get_latest_consensus_by_flavor(flav); + struct consensus_cache_entry_t *cached_consensus = NULL; + smartlist_t *diff_from_digests = NULL; + compress_method_t compression_used = NO_METHOD; + if (!parse_or_diff_from_header(&diff_from_digests, args->headers)) { + tor_assert(diff_from_digests); + cached_consensus = find_best_diff(diff_from_digests, flav, + args->compression_supported, + &compression_used); + SMARTLIST_FOREACH(diff_from_digests, uint8_t *, d, tor_free(d)); + smartlist_free(diff_from_digests); + } else { + cached_consensus = find_best_consensus(flav, + args->compression_supported, + &compression_used); + } - if (v && !networkstatus_consensus_reasonably_live(v, now)) { + if (cached_consensus && + !consensus_cache_entry_is_reasonably_live(cached_consensus, now)) { write_http_status_line(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(v, flavor, now); + warn_consensus_is_too_old(cached_consensus, flavor, now); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); tor_free(flavor); goto done; } - if (v && want_fps && - !client_likes_consensus(v, want_fps)) { + if (cached_consensus && want_fps && + !client_likes_consensus(cached_consensus, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); @@ -3792,37 +3851,30 @@ handle_get_current_consensus(dir_connection_t *conn, goto done; } - struct consensus_cache_entry_t *cached_diff = NULL; - smartlist_t *diff_from_digests = NULL; - compress_method_t compression_used = NO_METHOD; - if (!parse_or_diff_from_header(&diff_from_digests, args->headers)) { - tor_assert(diff_from_digests); - cached_diff = find_best_diff(diff_from_digests, flav, - args->compression_supported, - &compression_used); - SMARTLIST_FOREACH(diff_from_digests, uint8_t *, d, tor_free(d)); - smartlist_free(diff_from_digests); - } - conn->spool = smartlist_new(); clear_spool = 1; { spooled_resource_t *spooled; - if (cached_diff) { - spooled = spooled_resource_new_from_cache_entry(cached_diff); + if (cached_consensus) { + spooled = spooled_resource_new_from_cache_entry(cached_consensus); } else if (flavor) { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, (uint8_t*)flavor, strlen(flavor)); - compression_used = compress_method; + compression_used = ZLIB_METHOD; } else { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, NULL, 0); - compression_used = compress_method; + compression_used = ZLIB_METHOD; } tor_free(flavor); smartlist_add(conn->spool, spooled); } - lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; + + if (cached_consensus && + consensus_cache_entry_get_lifetime(cached_consensus, + &lifetime) != 0) { + lifetime = 0; + } if (!smartlist_len(conn->spool)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); @@ -3875,8 +3927,11 @@ handle_get_current_consensus(dir_connection_t *conn, write_http_response_header(conn, -1, compression_used, smartlist_len(conn->spool) == 1 ? lifetime : 0); - if (compress_method != NO_METHOD) - conn->compress_state = tor_compress_new(0, compress_method, + + // The compress_method requested was NO_METHOD, but we store the data + // compressed. Decompress them using `compression_used`. + if (compress_method == NO_METHOD) + conn->compress_state = tor_compress_new(0, compression_used, HIGH_COMPRESSION); /* Prime the connection with some data. */ -- cgit v1.2.3-54-g00ecf From 2f06345db3b6f85144c1d8a0b6ca55e2b1e243ce Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 14 May 2017 19:19:59 -0400 Subject: Move stub accessor functions a level higher, to consdiffmgr --- src/or/conscache.c | 59 ---------------------------------------------------- src/or/conscache.h | 8 ------- src/or/consdiffmgr.c | 51 +++++++++++++++++++++++++++++++++++++++++++++ src/or/consdiffmgr.h | 14 +++++++++++++ 4 files changed, 65 insertions(+), 67 deletions(-) (limited to 'src') diff --git a/src/or/conscache.c b/src/or/conscache.c index fd46cbe65e..5ffa129bbe 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -374,65 +374,6 @@ consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, return 0; } -/** Read the lifetime of cached object ent into lifetime. */ -int consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, - long *lifetime) -{ - if (BUG(ent->magic != CCE_MAGIC)) - return -1; // LCOV_EXCL_LINE - - tor_assert(lifetime); - - // FIXME(ahf): Fill out. - *lifetime = 0; - - return 0; -} - -/** Return non-zero if the cache object found in ent is - * reasonably live, otherwise return 0. Use now to pass the - * timestamp used for comparison. */ -int consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, - time_t now) -{ - if (BUG(ent->magic != CCE_MAGIC)) - return -1; // LCOV_EXCL_LINE - - // FIXME(ahf): Fill out. - (void)now; - - return 1; -} - -/** Read the set of voters from the cached object ent into out. */ -int consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, - smartlist_t *out) -{ - if (BUG(ent->magic != CCE_MAGIC)) - return -1; // LCOV_EXCL_LINE - - // FIXME(ahf): Fill out. - (void)out; - - return 0; -} - -/** Read the valid until timestamp from the cached object ent - * into out. */ -int consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, - time_t *out) -{ - if (BUG(ent->magic != CCE_MAGIC)) - return -1; // LCOV_EXCL_LINE - - tor_assert(out); - - // FIXME(ahf): Fill out. - *out = time(NULL); - - return 0; -} - /** * Unmap every mmap'd element of cache that has been unused * since cutoff. diff --git a/src/or/conscache.h b/src/or/conscache.h index fcc3f3e330..aef54201f0 100644 --- a/src/or/conscache.h +++ b/src/or/conscache.h @@ -52,14 +52,6 @@ void consensus_cache_entry_mark_for_aggressive_release( int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, const uint8_t **body_out, size_t *sz_out); -int consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, - long *lifetime); -int consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, - time_t now); -int consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, - smartlist_t *out); -int consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, - time_t *out); #ifdef TOR_UNIT_TESTS int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent); diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index a2e269323f..c208865170 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -1729,3 +1729,54 @@ consdiffmgr_enable_background_compression(void) background_compression = 1; } +/** Read the lifetime of cached object ent into lifetime. */ +int +consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, + long *lifetime) +{ + tor_assert(lifetime); + + // FIXME(ahf): Fill out. + *lifetime = 0; + + return 0; +} + +/** Return non-zero if the cache object found in ent is + * reasonably live, otherwise return 0. Use now to pass the + * timestamp used for comparison. */ +int +consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, + time_t now) +{ + // FIXME(ahf): Fill out. + (void)now; + + return 1; +} + +/** Read the set of voters from the cached object ent into out. */ +int +consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, + smartlist_t *out) +{ + // FIXME(ahf): Fill out. + (void)out; + + return 0; +} + +/** Read the valid until timestamp from the cached object ent + * into out. */ +int +consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, + time_t *out) +{ + tor_assert(out); + + // FIXME(ahf): Fill out. + *out = time(NULL); + + return 0; +} + diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index abe1ea9dc3..a8111dd7be 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -35,6 +35,20 @@ consdiff_status_t consdiffmgr_find_diff_from( const uint8_t *digest, size_t digestlen, compress_method_t method); + +int consensus_cache_entry_get_lifetime( + const struct consensus_cache_entry_t *ent, + long *lifetime); +int consensus_cache_entry_is_reasonably_live( + const struct consensus_cache_entry_t *ent, + time_t now); +int consensus_cache_entry_get_voters( + const struct consensus_cache_entry_t *ent, + smartlist_t *out); +int consensus_cache_entry_valid_until( + const struct consensus_cache_entry_t *ent, + time_t *out); + void consdiffmgr_rescan(void); int consdiffmgr_cleanup(void); void consdiffmgr_enable_background_compression(void); -- cgit v1.2.3-54-g00ecf From dcc533fb133646d81c2fa6632d6fb7f05f99f650 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 14 May 2017 19:43:41 -0400 Subject: Implement functions to expose valid/fresh-until and voters These still won't do anything till I get the values to be filled in. Also, I changed the API a little (with corresponding changes in directory.c) to match things that it's easier to store. --- src/or/consdiffmgr.c | 80 +++++++++++++++++++++++++++----------------------- src/or/consdiffmgr.h | 13 ++++---- src/or/directory.c | 54 +++++++++++++++------------------- src/or/networkstatus.c | 18 ++++++++---- src/or/networkstatus.h | 4 ++- 5 files changed, 88 insertions(+), 81 deletions(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index c208865170..d117acd731 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -32,6 +32,15 @@ /* The valid-after time for a consensus (or for the target consensus of a * diff), encoded as ISO UTC. */ #define LABEL_VALID_AFTER "consensus-valid-after" +/* The fresh-until time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_FRESH_UNTIL "consensus-fresh-until" +/* The valid-until time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_VALID_UNTIL "consensus-valid-until" +/* Comma-separated list of hex-encoded identity digests for the voting + * authorities. */ +#define LABEL_SIGNATORIES "consensus-signatories" /* A hex encoded SHA3 digest of the object, as compressed (if any) */ #define LABEL_SHA3_DIGEST "sha3-digest" /* A hex encoded SHA3 digest of the object before compression. */ @@ -1729,54 +1738,53 @@ consdiffmgr_enable_background_compression(void) background_compression = 1; } -/** Read the lifetime of cached object ent into lifetime. */ +/** Read the set of voters from the cached object ent into + * out, as a list of hex-encoded digests. Return 0 on success, + * -1 if no signatories were recorded. */ int -consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, - long *lifetime) +consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent, + smartlist_t *out) { - tor_assert(lifetime); - - // FIXME(ahf): Fill out. - *lifetime = 0; - + tor_assert(ent); + tor_assert(out); + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES); + if (s == NULL) + return -1; + smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0); return 0; } -/** Return non-zero if the cache object found in ent is - * reasonably live, otherwise return 0. Use now to pass the - * timestamp used for comparison. */ -int -consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, - time_t now) -{ - // FIXME(ahf): Fill out. - (void)now; - - return 1; -} - -/** Read the set of voters from the cached object ent into out. */ +/** Read the fresh-until time of cached object ent into *out + * and return 0, or return -1 if no such time was recorded. */ int -consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, - smartlist_t *out) +consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent, + time_t *out) { - // FIXME(ahf): Fill out. - (void)out; - - return 0; + tor_assert(ent); + tor_assert(out); + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL); + if (s == NULL || parse_iso_time_nospace(s, out) < 0) + return -1; + else + return 0; } -/** Read the valid until timestamp from the cached object ent - * into out. */ +/** Read the valid until timestamp from the cached object ent into + * *out and return 0, or return -1 if no such time was recorded. */ int -consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, - time_t *out) +consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent, + time_t *out) { + tor_assert(ent); tor_assert(out); - // FIXME(ahf): Fill out. - *out = time(NULL); - - return 0; + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL); + if (s == NULL || parse_iso_time_nospace(s, out) < 0) + return -1; + else + return 0; } diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index a8111dd7be..0325a00bf5 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -36,16 +36,13 @@ consdiff_status_t consdiffmgr_find_diff_from( size_t digestlen, compress_method_t method); -int consensus_cache_entry_get_lifetime( - const struct consensus_cache_entry_t *ent, - long *lifetime); -int consensus_cache_entry_is_reasonably_live( - const struct consensus_cache_entry_t *ent, - time_t now); -int consensus_cache_entry_get_voters( +int consensus_cache_entry_get_voter_id_digests( const struct consensus_cache_entry_t *ent, smartlist_t *out); -int consensus_cache_entry_valid_until( +int consensus_cache_entry_get_fresh_until( + const struct consensus_cache_entry_t *ent, + time_t *out); +int consensus_cache_entry_get_valid_until( const struct consensus_cache_entry_t *ent, time_t *out); diff --git a/src/or/directory.c b/src/or/directory.c index 946792b1b3..4fa4b3c040 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3382,47 +3382,35 @@ int client_likes_consensus(const struct consensus_cache_entry_t *ent, const char *want_url) { - smartlist_t *want_authorities = smartlist_new(); smartlist_t *voters = smartlist_new(); int need_at_least; int have = 0; + if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) { + return 1; // We don't know the voters; assume the client won't mind. */ + } + + smartlist_t *want_authorities = smartlist_new(); dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - if (consensus_cache_entry_get_voters(ent, voters) != 0) - goto done; - - SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { - char want_digest[DIGEST_LEN]; - size_t want_len = strlen(d)/2; - if (want_len > DIGEST_LEN) - want_len = DIGEST_LEN; + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) { - if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) - != (int) want_len) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Failed to decode requested authority digest %s.", escaped(d)); - continue; - }; - - SMARTLIST_FOREACH_BEGIN(voters, networkstatus_voter_info_t *, vi) { - if (smartlist_len(vi->sigs) && - tor_memeq(vi->identity_digest, want_digest, want_len)) { + SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) { + if (!strcasecmpstart(digest, want_digest)) { have++; break; }; - } SMARTLIST_FOREACH_END(vi); + } SMARTLIST_FOREACH_END(digest); /* early exit, if we already have enough */ if (have >= need_at_least) break; - } SMARTLIST_FOREACH_END(d); + } SMARTLIST_FOREACH_END(want_digest); - done: SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); - SMARTLIST_FOREACH(voters, networkstatus_voter_info_t *, v, tor_free(v)); + SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp)); smartlist_free(voters); return (have >= need_at_least); } @@ -3644,7 +3632,7 @@ warn_consensus_is_too_old(const struct consensus_cache_entry_t *cached_consensus time_t valid_until; char *dupes; - if (consensus_cache_entry_valid_until(cached_consensus, &valid_until)) + if (consensus_cache_entry_get_valid_until(cached_consensus, &valid_until)) return; if ((dupes = rate_limit_log(&warned, now))) { @@ -3832,9 +3820,17 @@ handle_get_current_consensus(dir_connection_t *conn, args->compression_supported, &compression_used); } + time_t fresh_until, valid_until; + int have_fresh_until = 0, have_valid_until = 0; + if (cached_consensus) { + have_fresh_until = + !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); + have_valid_until = + !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); + } - if (cached_consensus && - !consensus_cache_entry_is_reasonably_live(cached_consensus, now)) { + if (cached_consensus && have_valid_until && + !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { write_http_status_line(conn, 404, "Consensus is too old"); warn_consensus_is_too_old(cached_consensus, flavor, now); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); @@ -3870,11 +3866,7 @@ handle_get_current_consensus(dir_connection_t *conn, smartlist_add(conn->spool, spooled); } - if (cached_consensus && - consensus_cache_entry_get_lifetime(cached_consensus, - &lifetime) != 0) { - lifetime = 0; - } + lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; if (!smartlist_len(conn->spool)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index bd106fd9a6..fffd1078be 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1407,16 +1407,24 @@ networkstatus_get_live_consensus,(time_t now)) * Return 1 if the consensus is reasonably live, or 0 if it is too old. */ int -networkstatus_consensus_reasonably_live(networkstatus_t *consensus, time_t now) +networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, + time_t now) { -#define REASONABLY_LIVE_TIME (24*60*60) if (BUG(!consensus)) return 0; - if (now <= consensus->valid_until + REASONABLY_LIVE_TIME) - return 1; + return networkstatus_valid_until_is_reasonably_live(consensus->valid_until, + now); +} - return 0; +/** As networkstatus_consensus_reasonably_live, but takes a valid_until + * time rather than an entire consensus. */ +int +networkstatus_valid_until_is_reasonably_live(time_t valid_until, + time_t now) +{ +#define REASONABLY_LIVE_TIME (24*60*60) + return (now <= valid_until + REASONABLY_LIVE_TIME); } /* XXXX remove this in favor of get_live_consensus. But actually, diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 37a5a3b7a0..e774c4d266 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -81,8 +81,10 @@ MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void)); MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor, (consensus_flavor_t f)); MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); -int networkstatus_consensus_reasonably_live(networkstatus_t *consensus, +int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); +int networkstatus_valid_until_is_reasonably_live(time_t valid_until, + time_t now); networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, int flavor); MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now)); -- cgit v1.2.3-54-g00ecf From fd1190581dd1d33e32ae97bf705cd4b470911f09 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 14 May 2017 20:01:36 -0400 Subject: Store fresh/valid-until and signatories values on all consensus objects. --- src/or/consdiffmgr.c | 49 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index d117acd731..fef5f68505 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -175,8 +175,7 @@ static consdiff_cfg_t consdiff_cfg = { static int consdiffmgr_ensure_space_for_files(int n); static int consensus_queue_compression_work(const char *consensus, - consensus_flavor_t flavor, - time_t valid_after); + const networkstatus_t *as_parsed); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); static void consdiffmgr_set_cache_flags(void); @@ -492,7 +491,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - return consensus_queue_compression_work(consensus, flavor, valid_after); + return consensus_queue_compression_work(consensus, as_parsed); } /** @@ -1591,7 +1590,7 @@ typedef struct consensus_compress_worker_job_t { char *consensus; size_t consensus_len; consensus_flavor_t flavor; - time_t valid_after; + config_line_t *labels_in; compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)]; } consensus_compress_worker_job_t; @@ -1604,6 +1603,7 @@ consensus_compress_worker_job_free(consensus_compress_worker_job_t *job) if (!job) return; tor_free(job->consensus); + config_free_lines(job->labels_in); unsigned u; for (u = 0; u < n_consensus_compression_methods(); ++u) { config_free_lines(job->out[u].labels); @@ -1623,12 +1623,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_) consensus_flavor_t flavor = job->flavor; const char *consensus = job->consensus; size_t bodylen = job->consensus_len; - time_t valid_after = job->valid_after; - - config_line_t *labels = NULL; - char formatted_time[ISO_TIME_LEN+1]; - format_iso_time_nospace(formatted_time, valid_after); + config_line_t *labels = config_lines_dup(job->labels_in); const char *flavname = networkstatus_get_flavor_name(flavor); cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, @@ -1645,7 +1641,6 @@ consensus_compress_worker_threadfn(void *state_, void *work_) end - start); } config_line_prepend(&labels, LABEL_FLAVOR, flavname); - config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); compress_multiple(job->out, @@ -1701,14 +1696,40 @@ static int background_compression = 0; */ static int consensus_queue_compression_work(const char *consensus, - consensus_flavor_t flavor, - time_t valid_after) + const networkstatus_t *as_parsed) { + tor_assert(consensus); + tor_assert(as_parsed); + consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); job->consensus = tor_strdup(consensus); job->consensus_len = strlen(consensus); - job->flavor = flavor; - job->valid_after = valid_after; + job->flavor = as_parsed->flavor; + + char va_str[ISO_TIME_LEN+1]; + char vu_str[ISO_TIME_LEN+1]; + char fu_str[ISO_TIME_LEN+1]; + format_iso_time_nospace(va_str, as_parsed->valid_after); + format_iso_time_nospace(fu_str, as_parsed->fresh_until); + format_iso_time_nospace(vu_str, as_parsed->valid_until); + config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str); + config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str); + config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str); + if (as_parsed->voters) { + smartlist_t *hexvoters = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(as_parsed->voters, + networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) == 0) + continue; // didn't sign. + char d[HEX_DIGEST_LEN+1]; + base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN); + smartlist_add_strdup(hexvoters, d); + } SMARTLIST_FOREACH_END(vi); + char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL); + config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers); + tor_free(signers); + SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp)); + } if (background_compression) { workqueue_entry_t *work; -- cgit v1.2.3-54-g00ecf From 7591518d163de1367cd21e1fc610960ab83993e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 14 May 2017 20:06:46 -0400 Subject: Copy valid/fresh-until and signatories values into diffs. --- src/or/consdiffmgr.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src') diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index fef5f68505..910f842070 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -1348,6 +1348,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) const char *lv_to_valid_after = consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER); + const char *lv_to_fresh_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL); + const char *lv_to_valid_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL); + const char *lv_to_signatories = + consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES); const char *lv_from_valid_after = consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER); const char *lv_from_digest = @@ -1417,6 +1423,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) job->out[0].bodylen = difflen; config_line_t *common_labels = NULL; + if (lv_to_valid_until) + config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until); + if (lv_to_fresh_until) + config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until); + if (lv_to_signatories) + config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories); cdm_labels_prepend_sha3(&common_labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, job->out[0].body, -- cgit v1.2.3-54-g00ecf From ae33deb91d204da0689f182a0899f9f11a5bb77a Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 15 May 2017 12:09:07 +0200 Subject: Check for best consensus when no consensusdiff was found. This patch ensures that we use the current consensus in the case where no consensus diff was found or a consensus diff wasn't requested. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 4fa4b3c040..8ed1305b06 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3815,11 +3815,14 @@ handle_get_current_consensus(dir_connection_t *conn, &compression_used); SMARTLIST_FOREACH(diff_from_digests, uint8_t *, d, tor_free(d)); smartlist_free(diff_from_digests); - } else { + } + + if (! cached_consensus) { cached_consensus = find_best_consensus(flav, args->compression_supported, &compression_used); } + time_t fresh_until, valid_until; int have_fresh_until = 0, have_valid_until = 0; if (cached_consensus) { -- cgit v1.2.3-54-g00ecf From fade313ba30279b76794368b4ff9e97be6bb159d Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 15 May 2017 12:34:12 +0200 Subject: Fix too wide line from `make check-spaces`. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 8ed1305b06..dbee0b3fc0 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3618,12 +3618,12 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) return 0; } -/** Warn that the cached consensus cached_consensus of type +/** Warn that the cached consensus consensus of type * flavor is too old and will not be served to clients. Rate-limit the * warning to avoid logging an entry on every request. */ static void -warn_consensus_is_too_old(const struct consensus_cache_entry_t *cached_consensus, +warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, const char *flavor, time_t now) { #define TOO_OLD_WARNING_INTERVAL (60*60) @@ -3632,7 +3632,7 @@ warn_consensus_is_too_old(const struct consensus_cache_entry_t *cached_consensus time_t valid_until; char *dupes; - if (consensus_cache_entry_get_valid_until(cached_consensus, &valid_until)) + if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) return; if ((dupes = rate_limit_log(&warned, now))) { -- cgit v1.2.3-54-g00ecf From 8d730af0f7cd041cd83c237ca0f102ff076c18b0 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 15 May 2017 14:50:04 +0200 Subject: Remove old consensus fetching code from handle_get_current_consensus(). This patch removes the calls to spooled_resource_new() when trying to download the consensus. All calls should now be going through the consdiff manager. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index dbee0b3fc0..6504e2e7ad 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3856,17 +3856,9 @@ handle_get_current_consensus(dir_connection_t *conn, spooled_resource_t *spooled; if (cached_consensus) { spooled = spooled_resource_new_from_cache_entry(cached_consensus); - } else if (flavor) { - spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, - (uint8_t*)flavor, strlen(flavor)); - compression_used = ZLIB_METHOD; - } else { - spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, - NULL, 0); - compression_used = ZLIB_METHOD; + smartlist_add(conn->spool, spooled); } tor_free(flavor); - smartlist_add(conn->spool, spooled); } lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; -- cgit v1.2.3-54-g00ecf From 008194035fa8ab3712dead9152622de8509e8cf3 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 15 May 2017 15:13:28 +0200 Subject: Handle non-compressed requests gracefully. This patch makes us use FALLBACK_COMPRESS_METHOD to try to fetch an object from the consensus diff manager in case no mutually supported result was found. This object, if found, is then decompressed using the spooling system to the client. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index 6504e2e7ad..cf6f0db977 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3677,6 +3677,13 @@ parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) return 0; } +/** Fallback compression method. The fallback compression method is used in + * case a client requests a non-compressed document. We only store compressed + * documents, so we use this compression method to fetch the document and let + * the spooling system do the streaming decompression. + */ +#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD + /** * Try to find the best consensus diff possible in order to serve a client * request for a diff from one of the consensuses in digests to the @@ -3706,6 +3713,16 @@ find_best_diff(const smartlist_t *digests, int flav, } } } SMARTLIST_FOREACH_END(diff_from); + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from, + DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + } SMARTLIST_FOREACH_END(diff_from); + return NULL; } @@ -3735,6 +3752,13 @@ find_best_consensus(int flav, } } + if (consdiffmgr_find_consensus(&result, flav, + FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + return NULL; } @@ -3911,12 +3935,15 @@ handle_get_current_consensus(dir_connection_t *conn, } clear_spool = 0; + + // The compress_method might have been NO_METHOD, but we store the data + // compressed. Decompress them using `compression_used`. See fallback code in + // find_best_consensus() and find_best_diff(). write_http_response_header(conn, -1, - compression_used, + compress_method == NO_METHOD ? + NO_METHOD : compression_used, smartlist_len(conn->spool) == 1 ? lifetime : 0); - // The compress_method requested was NO_METHOD, but we store the data - // compressed. Decompress them using `compression_used`. if (compress_method == NO_METHOD) conn->compress_state = tor_compress_new(0, compression_used, HIGH_COMPRESSION); -- cgit v1.2.3-54-g00ecf From 65d940844887edb3285a35dd535f2fa68b873650 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 15 May 2017 10:30:35 -0400 Subject: dir_handle_get: repair two test cases, note the fixes for 3 others --- src/test/test_dir_handle_get.c | 49 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index c98938b2db..b2e9b8b5fa 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -12,6 +12,7 @@ #include "or.h" #include "config.h" #include "connection.h" +#include "consdiffmgr.h" #include "directory.h" #include "test.h" #include "compress.h" @@ -465,6 +466,8 @@ init_mock_options(void) mock_options = tor_malloc(sizeof(or_options_t)); memset(mock_options, 0, sizeof(or_options_t)); mock_options->TestingTorNetwork = 1; + mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp")); + check_private_dir(mock_options->DataDirectory, CPD_CREATE, NULL); } static const or_options_t * @@ -501,14 +504,6 @@ test_dir_handle_get_micro_d(void *data) /* SETUP */ init_mock_options(); - const char *fn = get_fname("dir_handle_datadir_test1"); - mock_options->DataDirectory = tor_strdup(fn); - -#ifdef _WIN32 - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory)); -#else - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700)); -#endif /* Add microdesc to cache */ crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256); @@ -568,14 +563,6 @@ test_dir_handle_get_micro_d_server_busy(void *data) /* SETUP */ init_mock_options(); - const char *fn = get_fname("dir_handle_datadir_test2"); - mock_options->DataDirectory = tor_strdup(fn); - -#ifdef _WIN32 - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory)); -#else - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700)); -#endif /* Add microdesc to cache */ crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256); @@ -1619,6 +1606,8 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d) (void) d; /* init mock */ + // XXXX Instead of mocking this, we need to mock + // XXXX consensus_cache_entry_get_voter_id_digests(). mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); mock_ns_val->flavor = FLAV_NS; mock_ns_val->voters = smartlist_new(); @@ -1684,6 +1673,8 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_found(void* data) tt_int_op(0, OP_EQ, directory_handle_command_get(conn, GET("/tor/status-vote/current/consensus-ns"), NULL, 0)); + // XXXX This fails with 503 instead of 404; I think that's a regression in + // XXXX our behavior. fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, NULL, NULL, 1, 0); tt_assert(header); @@ -1709,6 +1700,8 @@ test_dir_handle_get_status_vote_current_consensus_too_old(void *data) char *header = NULL; (void)data; + // XXXX Instead of mocking this, we need to mock + // XXXX consensus_cache_entry_get_valid_until. mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); mock_ns_val->flavor = FLAV_MICRODESC; mock_ns_val->valid_until = time(NULL) - (60 * 60 * 24) - 1; @@ -1772,16 +1765,26 @@ static void status_vote_current_consensus_ns_test(char **header, char **body, size_t *body_len) { - common_digests_t digests; - uint8_t sha3[DIGEST256_LEN]; dir_connection_t *conn = NULL; #define NETWORK_STATUS "some network status string" +#if 0 + common_digests_t digests; + uint8_t sha3[DIGEST256_LEN]; memset(&digests, 0x60, sizeof(digests)); memset(sha3, 0x06, sizeof(sha3)); dirserv_set_cached_consensus_networkstatus(NETWORK_STATUS, "ns", &digests, sha3, time(NULL)); +#endif + networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t)); + ns->type = NS_TYPE_CONSENSUS; + ns->flavor = FLAV_NS; + ns->valid_after = time(NULL) - 1800; + ns->fresh_until = time(NULL) - 900; + ns->valid_until = time(NULL) - 60; + consdiffmgr_add_consensus(NETWORK_STATUS, ns); + networkstatus_vote_free(ns); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -2592,11 +2595,11 @@ struct testcase_t dir_handle_get_tests[] = { DIR_HANDLE_CMD(status_vote_current_authority, 0), DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0), DIR_HANDLE_CMD(status_vote_next_authority, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_too_old, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns, 0), + DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_ns, TT_FORK), DIR_HANDLE_CMD(status_vote_current_d_not_found, 0), DIR_HANDLE_CMD(status_vote_next_d_not_found, 0), DIR_HANDLE_CMD(status_vote_d, 0), -- cgit v1.2.3-54-g00ecf From 9e3f3041139f60c5d109aff345bc560490329b10 Mon Sep 17 00:00:00 2001 From: Alexander Færøy Date: Mon, 15 May 2017 20:32:26 +0200 Subject: Fix dir_handle_get/... test-cases for prop#278 support. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 6 ------ src/test/test_dir_handle_get.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/or/directory.c b/src/or/directory.c index cf6f0db977..d2ac42c3ff 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3887,12 +3887,6 @@ handle_get_current_consensus(dir_connection_t *conn, lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; - if (!smartlist_len(conn->spool)) { /* we failed to create/cache cp */ - write_http_status_line(conn, 503, "Network status object unavailable"); - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); - goto done; - } - size_t size_guess = 0; int n_expired = 0; dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index b2e9b8b5fa..7d40db4543 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1606,12 +1606,15 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d) (void) d; /* init mock */ - // XXXX Instead of mocking this, we need to mock - // XXXX consensus_cache_entry_get_voter_id_digests(). mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); mock_ns_val->flavor = FLAV_NS; + mock_ns_val->type = NS_TYPE_CONSENSUS; mock_ns_val->voters = smartlist_new(); - mock_ns_val->valid_until = time(NULL); + mock_ns_val->valid_after = time(NULL) - 1800; + mock_ns_val->valid_until = time(NULL) - 60; + + #define NETWORK_STATUS "some network status string" + consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val); /* init mock */ init_mock_options(); @@ -1673,8 +1676,6 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_found(void* data) tt_int_op(0, OP_EQ, directory_handle_command_get(conn, GET("/tor/status-vote/current/consensus-ns"), NULL, 0)); - // XXXX This fails with 503 instead of 404; I think that's a regression in - // XXXX our behavior. fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, NULL, NULL, 1, 0); tt_assert(header); @@ -1700,18 +1701,24 @@ test_dir_handle_get_status_vote_current_consensus_too_old(void *data) char *header = NULL; (void)data; - // XXXX Instead of mocking this, we need to mock - // XXXX consensus_cache_entry_get_valid_until. mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); + mock_ns_val->type = NS_TYPE_CONSENSUS; mock_ns_val->flavor = FLAV_MICRODESC; - mock_ns_val->valid_until = time(NULL) - (60 * 60 * 24) - 1; + mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800); + mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900); + mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20); + + #define NETWORK_STATUS "some network status string" + consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val); init_mock_options(); + MOCK(get_options, mock_get_options); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); MOCK(networkstatus_get_latest_consensus_by_flavor, mock_ns_get_by_flavor); conn = new_dir_conn(); + TO_CONN(conn)->address = tor_strdup("127.0.0.1"); setup_capture_of_logs(LOG_WARN); @@ -1727,6 +1734,17 @@ test_dir_handle_get_status_vote_current_consensus_too_old(void *data) tor_free(header); teardown_capture_of_logs(); + tor_free(mock_ns_val); + + mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); + mock_ns_val->type = NS_TYPE_CONSENSUS; + mock_ns_val->flavor = FLAV_NS; + mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800); + mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900); + mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20); + + #define NETWORK_STATUS "some network status string" + consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val); setup_capture_of_logs(LOG_WARN); -- cgit v1.2.3-54-g00ecf