diff options
Diffstat (limited to 'src/or')
159 files changed, 12026 insertions, 3439 deletions
diff --git a/src/or/Makefile.nmake b/src/or/Makefile.nmake index 2ac98cd372..429ae67858 100644 --- a/src/or/Makefile.nmake +++ b/src/or/Makefile.nmake @@ -14,6 +14,7 @@ LIBTOR_OBJECTS = \ addressmap.obj \ buffers.obj \ channel.obj \ + channelpadding.obj \ channeltls.obj \ circpathbias.obj \ circuitbuild.obj \ diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 85a6434f4a..c92af38254 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/addressmap.h b/src/or/addressmap.h index 67648d0518..80f453b4c1 100644 --- a/src/or/addressmap.h +++ b/src/or/addressmap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ADDRESSMAP_H diff --git a/src/or/bridges.c b/src/or/bridges.c index 88154c6c8a..0818fb0812 100644 --- a/src/or/bridges.c +++ b/src/or/bridges.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -547,6 +547,7 @@ static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { const or_options_t *options = get_options(); + circuit_guard_state_t *guard_state = NULL; if (connection_get_by_type_addr_port_purpose( CONN_TYPE_DIR, &bridge->addr, bridge->port, @@ -570,12 +571,23 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } - directory_initiate_command(&bridge->addr, bridge->port, - NULL, 0, /*no dirport*/ - bridge->identity, - DIR_PURPOSE_FETCH_SERVERDESC, - ROUTER_PURPOSE_BRIDGE, - DIRIND_ONEHOP, "authority.z", NULL, 0, 0); + tor_addr_port_t bridge_addrport; + memcpy(&bridge_addrport.addr, &bridge->addr, sizeof(tor_addr_t)); + bridge_addrport.port = bridge->port; + + guard_state = get_guard_state_for_bridge_desc_fetch(bridge->identity); + + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); + directory_request_set_or_addr_port(req, &bridge_addrport); + directory_request_set_directory_id_digest(req, bridge->identity); + directory_request_set_router_purpose(req, ROUTER_PURPOSE_BRIDGE); + directory_request_set_resource(req, "authority.z"); + if (guard_state) { + directory_request_set_guard_state(req, guard_state); + } + directory_initiate_request(req); + directory_request_free(req); } /** Fetching the bridge descriptor from the bridge authority returned a diff --git a/src/or/bridges.h b/src/or/bridges.h index 27ea5e197c..3bfc782f9a 100644 --- a/src/or/bridges.h +++ b/src/or/bridges.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/buffers.c b/src/or/buffers.c index 603da1bb6e..3692ed4d08 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -83,7 +83,11 @@ static int parse_socks_client(const uint8_t *data, size_t datalen, #define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0]) /* We leave this many NUL bytes at the end of the buffer. */ +#ifdef DISABLE_MEMORY_SENTINELS +#define SENTINEL_LEN 0 +#else #define SENTINEL_LEN 4 +#endif /* Header size plus NUL bytes at the end */ #define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN) @@ -97,18 +101,22 @@ static int parse_socks_client(const uint8_t *data, size_t datalen, #define DEBUG_SENTINEL -#ifdef DEBUG_SENTINEL +#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS) #define DBG_S(s) s #else #define DBG_S(s) (void)0 #endif +#ifdef DISABLE_MEMORY_SENTINELS +#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL +#else #define CHUNK_SET_SENTINEL(chunk, alloclen) do { \ uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen]; \ DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]); \ DBG_S(tor_assert(a == b)); \ memset(a,0,SENTINEL_LEN); \ } while (0) +#endif /** Return the next character in <b>chunk</b> onto which data can be appended. * If the chunk is full, this might be off the end of chunk->mem. */ @@ -1311,7 +1319,7 @@ fetch_from_buf_http(buf_t *buf, /** * Wait this many seconds before warning the user about using SOCKS unsafely - * again (requires that WarnUnsafeSocks is turned on). */ + * again. */ #define SOCKS_WARN_INTERVAL 5 /** Warn that the user application has made an unsafe socks request using @@ -1323,9 +1331,6 @@ log_unsafe_socks_warning(int socks_protocol, const char *address, { static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL); - const or_options_t *options = get_options(); - if (! options->WarnUnsafeSocks) - return; if (safe_socks) { log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP, "Your application (using socks%d to port %d) is giving " @@ -1710,6 +1715,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, return -1; } tor_assert(0); + break; case 4: { /* socks4 */ enum {socks4, socks4a} socks4_prot = socks4a; const char *authstart, *authend; @@ -2080,13 +2086,13 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) } /** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the - * zlib state <b>state</b>, appending the result to <b>buf</b>. If + * compression state <b>state</b>, appending the result to <b>buf</b>. If * <b>done</b> is true, flush the data in the state and finish the * compression/uncompression. Return -1 on failure, 0 on success. */ int -write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, - int done) +write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + int done) { char *next; size_t old_avail, avail; @@ -2100,20 +2106,22 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, } next = CHUNK_WRITE_PTR(buf->tail); avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); - switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) { - case TOR_ZLIB_DONE: + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: over = 1; break; - case TOR_ZLIB_ERR: + case TOR_COMPRESS_ERROR: return -1; - case TOR_ZLIB_OK: + case TOR_COMPRESS_OK: if (data_len == 0) over = 1; break; - case TOR_ZLIB_BUF_FULL: + case TOR_COMPRESS_BUFFER_FULL: if (avail) { - /* Zlib says we need more room (ZLIB_BUF_FULL). Start a new chunk - * automatically, whether were going to or not. */ + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ need_new_chunk = 1; } break; diff --git a/src/or/buffers.h b/src/or/buffers.h index bb53b3bbff..23b58a571a 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -36,8 +36,8 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, int done); +int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, int done); int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); int fetch_from_buf(char *string, size_t string_len, buf_t *buf); int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); diff --git a/src/or/channel.c b/src/or/channel.c index 45f1602ab2..df6d7d3423 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -49,6 +49,7 @@ #include "or.h" #include "channel.h" #include "channeltls.h" +#include "channelpadding.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -63,6 +64,7 @@ #include "router.h" #include "routerlist.h" #include "scheduler.h" +#include "compat_time.h" /* Global lists of channels */ @@ -84,6 +86,28 @@ static smartlist_t *active_listeners = NULL; /* All channel_listener_t instances in LISTENING state */ static smartlist_t *finished_listeners = NULL; +/** Map from channel->global_identifier to channel. Contains the same + * elements as all_channels. */ +static HT_HEAD(channel_gid_map, channel_s) channel_gid_map = HT_INITIALIZER(); + +static unsigned +channel_id_hash(const channel_t *chan) +{ + return (unsigned) chan->global_identifier; +} +static int +channel_id_eq(const channel_t *a, const channel_t *b) +{ + return a->global_identifier == b->global_identifier; +} +HT_PROTOTYPE(channel_gid_map, channel_s, gidmap_node, + channel_id_hash, channel_id_eq) +HT_GENERATE2(channel_gid_map, channel_s, gidmap_node, + channel_id_hash, channel_id_eq, + 0.6, tor_reallocarray_, tor_free_) + +HANDLE_IMPL(channel, channel_s,) + /* Counter for ID numbers */ static uint64_t n_channels_allocated = 0; /* @@ -429,6 +453,7 @@ void channel_register(channel_t *chan) { tor_assert(chan); + tor_assert(chan->global_identifier); /* No-op if already registered */ if (chan->registered) return; @@ -443,6 +468,8 @@ channel_register(channel_t *chan) /* Make sure we have all_channels, then add it */ if (!all_channels) all_channels = smartlist_new(); smartlist_add(all_channels, chan); + channel_t *oldval = HT_REPLACE(channel_gid_map, &channel_gid_map, chan); + tor_assert(! oldval); /* Is it finished? */ if (CHANNEL_FINISHED(chan)) { @@ -498,7 +525,9 @@ channel_unregister(channel_t *chan) } /* Get it out of all_channels */ - if (all_channels) smartlist_remove(all_channels, chan); + if (all_channels) smartlist_remove(all_channels, chan); + channel_t *oldval = HT_REMOVE(channel_gid_map, &channel_gid_map, chan); + tor_assert(oldval == NULL || oldval == chan); /* Mark it as unregistered */ chan->registered = 0; @@ -533,7 +562,7 @@ channel_listener_register(channel_listener_t *chan_l) channel_listener_state_to_string(chan_l->state), chan_l->state); - /* Make sure we have all_channels, then add it */ + /* Make sure we have all_listeners, then add it */ if (!all_listeners) all_listeners = smartlist_new(); smartlist_add(all_listeners, chan_l); @@ -578,7 +607,7 @@ channel_listener_unregister(channel_listener_t *chan_l) if (active_listeners) smartlist_remove(active_listeners, chan_l); } - /* Get it out of all_channels */ + /* Get it out of all_listeners */ if (all_listeners) smartlist_remove(all_listeners, chan_l); /* Mark it as unregistered */ @@ -719,15 +748,13 @@ channel_remove_from_digest_map(channel_t *chan) channel_t * channel_find_by_global_id(uint64_t global_identifier) { + channel_t lookup; channel_t *rv = NULL; - if (all_channels && smartlist_len(all_channels) > 0) { - SMARTLIST_FOREACH_BEGIN(all_channels, channel_t *, curr) { - if (curr->global_identifier == global_identifier) { - rv = curr; - break; - } - } SMARTLIST_FOREACH_END(curr); + lookup.global_identifier = global_identifier; + rv = HT_FIND(channel_gid_map, &channel_gid_map, &lookup); + if (rv) { + tor_assert(rv->global_identifier == global_identifier); } return rv; @@ -809,6 +836,83 @@ channel_next_with_rsa_identity(channel_t *chan) } /** + * Relays run this once an hour to look over our list of channels to other + * relays. It prints out some statistics if there are multiple connections + * to many relays. + * + * This function is similar to connection_or_set_bad_connections(), + * and probably could be adapted to replace it, if it was modified to actually + * take action on any of these connections. + */ +void +channel_check_for_duplicates(void) +{ + channel_idmap_entry_t **iter; + channel_t *chan; + int total_relay_connections = 0, total_relays = 0, total_canonical = 0; + int total_half_canonical = 0; + int total_gt_one_connection = 0, total_gt_two_connections = 0; + int total_gt_four_connections = 0; + + HT_FOREACH(iter, channel_idmap, &channel_identity_map) { + int connections_to_relay = 0; + + /* Only consider relay connections */ + if (!connection_or_digest_is_known_relay((char*)(*iter)->digest)) + continue; + + total_relays++; + + for (chan = TOR_LIST_FIRST(&(*iter)->channel_list); chan; + chan = channel_next_with_rsa_identity(chan)) { + + if (CHANNEL_CONDEMNED(chan) || !CHANNEL_IS_OPEN(chan)) + continue; + + connections_to_relay++; + total_relay_connections++; + + if (chan->is_canonical(chan, 0)) total_canonical++; + + if (!chan->is_canonical_to_peer && chan->is_canonical(chan, 0) + && chan->is_canonical(chan, 1)) { + total_half_canonical++; + } + } + + if (connections_to_relay > 1) total_gt_one_connection++; + if (connections_to_relay > 2) total_gt_two_connections++; + if (connections_to_relay > 4) total_gt_four_connections++; + } + +#define MIN_RELAY_CONNECTIONS_TO_WARN 5 + + /* If we average 1.5 or more connections per relay, something is wrong */ + if (total_relays > MIN_RELAY_CONNECTIONS_TO_WARN && + total_relay_connections >= 1.5*total_relays) { + log_notice(LD_OR, + "Your relay has a very large number of connections to other relays. " + "Is your outbound address the same as your relay address? " + "Found %d connections to %d relays. Found %d current canonical " + "connections, in %d of which we were a non-canonical peer. " + "%d relays had more than 1 connection, %d had more than 2, and " + "%d had more than 4 connections.", + total_relay_connections, total_relays, total_canonical, + total_half_canonical, total_gt_one_connection, + total_gt_two_connections, total_gt_four_connections); + } else { + log_info(LD_OR, "Performed connection pruning. " + "Found %d connections to %d relays. Found %d current canonical " + "connections, in %d of which we were a non-canonical peer. " + "%d relays had more than 1 connection, %d had more than 2, and " + "%d had more than 4 connections.", + total_relay_connections, total_relays, total_canonical, + total_half_canonical, total_gt_one_connection, + total_gt_two_connections, total_gt_four_connections); + } +} + +/** * Initialize a channel * * This function should be called by subclasses to set up some per-channel @@ -822,7 +926,7 @@ channel_init(channel_t *chan) tor_assert(chan); /* Assign an ID and bump the counter */ - chan->global_identifier = n_channels_allocated++; + chan->global_identifier = ++n_channels_allocated; /* Init timestamp */ chan->timestamp_last_had_circuits = time(NULL); @@ -861,7 +965,7 @@ channel_init_listener(channel_listener_t *chan_l) tor_assert(chan_l); /* Assign an ID and bump the counter */ - chan_l->global_identifier = n_channels_allocated++; + chan_l->global_identifier = ++n_channels_allocated; /* Timestamp it */ channel_listener_timestamp_created(chan_l); @@ -898,6 +1002,11 @@ channel_free(channel_t *chan) circuitmux_set_policy(chan->cmux, NULL); } + /* Remove all timers and associated handle entries now */ + timer_free(chan->padding_timer); + channel_handle_free(chan->timer_handle); + channel_handles_clear(chan); + /* Call a free method if there is one */ if (chan->free_fn) chan->free_fn(chan); @@ -976,6 +1085,11 @@ channel_force_free(channel_t *chan) circuitmux_set_policy(chan->cmux, NULL); } + /* Remove all timers and associated handle entries now */ + timer_free(chan->padding_timer); + channel_handle_free(chan->timer_handle); + channel_handles_clear(chan); + /* Call a free method if there is one */ if (chan->free_fn) chan->free_fn(chan); @@ -2595,6 +2709,19 @@ channel_do_open_actions(channel_t *chan) } } + /* Disable or reduce padding according to user prefs. */ + if (chan->padding_enabled || get_options()->ConnectionPadding == 1) { + if (!get_options()->ConnectionPadding) { + channelpadding_disable_padding_on_channel(chan); + } + + /* Padding can be forced and/or reduced by clients, regardless of if + * the channel supports it */ + if (get_options()->ReducedConnectionPadding) { + channelpadding_reduce_padding_on_channel(chan); + } + } + circuit_n_chan_done(chan, 1, close_origin_circuits); } @@ -3232,6 +3359,11 @@ channel_free_all(void) /* Geez, anything still left over just won't die ... let it leak then */ HT_CLEAR(channel_idmap, &channel_identity_map); + /* Same with channel_gid_map */ + log_debug(LD_CHANNEL, + "Freeing channel_gid_map"); + HT_CLEAR(channel_gid_map, &channel_gid_map); + log_debug(LD_CHANNEL, "Done cleaning up after channels"); } @@ -3267,22 +3399,20 @@ channel_connect(const tor_addr_t *addr, uint16_t port, */ int -channel_is_better(time_t now, channel_t *a, channel_t *b, - int forgive_new_connections) +channel_is_better(channel_t *a, channel_t *b) { - int a_grace, b_grace; int a_is_canonical, b_is_canonical; - int a_has_circs, b_has_circs; - - /* - * Do not definitively deprecate a new channel with no circuits on it - * until this much time has passed. - */ -#define NEW_CHAN_GRACE_PERIOD (15*60) tor_assert(a); tor_assert(b); + /* If one channel is bad for new circuits, and the other isn't, + * use the one that is still good. */ + if (!channel_is_bad_for_new_circs(a) && channel_is_bad_for_new_circs(b)) + return 1; + if (channel_is_bad_for_new_circs(a) && !channel_is_bad_for_new_circs(b)) + return 0; + /* Check if one is canonical and the other isn't first */ a_is_canonical = channel_is_canonical(a); b_is_canonical = channel_is_canonical(b); @@ -3290,26 +3420,31 @@ channel_is_better(time_t now, channel_t *a, channel_t *b, if (a_is_canonical && !b_is_canonical) return 1; if (!a_is_canonical && b_is_canonical) return 0; + /* Check if we suspect that one of the channels will be preferred + * by the peer */ + if (a->is_canonical_to_peer && !b->is_canonical_to_peer) return 1; + if (!a->is_canonical_to_peer && b->is_canonical_to_peer) return 0; + /* - * Okay, if we're here they tied on canonicity. Next we check if - * they have any circuits, and if one does and the other doesn't, - * we prefer the one that does, unless we are forgiving and the - * one that has no circuits is in its grace period. + * Okay, if we're here they tied on canonicity, the prefer the older + * connection, so that the adversary can't create a new connection + * and try to switch us over to it (which will leak information + * about long-lived circuits). Additionally, switching connections + * too often makes us more vulnerable to attacks like Torscan and + * passive netflow-based equivalents. + * + * Connections will still only live for at most a week, due to + * the check in connection_or_group_set_badness() against + * TIME_BEFORE_OR_CONN_IS_TOO_OLD, which marks old connections as + * unusable for new circuits after 1 week. That check sets + * is_bad_for_new_circs, which is checked in channel_get_for_extend(). + * + * We check channel_is_bad_for_new_circs() above here anyway, for safety. */ + if (channel_when_created(a) < channel_when_created(b)) return 1; + else if (channel_when_created(a) > channel_when_created(b)) return 0; - a_has_circs = (channel_num_circuits(a) > 0); - b_has_circs = (channel_num_circuits(b) > 0); - a_grace = (forgive_new_connections && - (now < channel_when_created(a) + NEW_CHAN_GRACE_PERIOD)); - b_grace = (forgive_new_connections && - (now < channel_when_created(b) + NEW_CHAN_GRACE_PERIOD)); - - if (a_has_circs && !b_has_circs && !b_grace) return 1; - if (!a_has_circs && b_has_circs && !a_grace) return 0; - - /* They tied on circuits too; just prefer whichever is newer */ - - if (channel_when_created(a) > channel_when_created(b)) return 1; + if (channel_num_circuits(a) > channel_num_circuits(b)) return 1; else return 0; } @@ -3334,7 +3469,6 @@ channel_get_for_extend(const char *rsa_id_digest, channel_t *chan, *best = NULL; int n_inprogress_goodaddr = 0, n_old = 0; int n_noncanonical = 0, n_possible = 0; - time_t now = approx_time(); tor_assert(msg_out); tor_assert(launch_out); @@ -3404,7 +3538,7 @@ channel_get_for_extend(const char *rsa_id_digest, continue; } - if (channel_is_better(now, chan, best, 0)) + if (channel_is_better(chan, best)) best = chan; } @@ -4186,8 +4320,12 @@ channel_timestamp_active(channel_t *chan) time_t now = time(NULL); tor_assert(chan); + chan->timestamp_xfer_ms = monotime_coarse_absolute_msec(); chan->timestamp_active = now; + + /* Clear any potential netflow padding timer. We're active */ + chan->next_padding_time_ms = 0; } /** @@ -4270,11 +4408,14 @@ void channel_timestamp_recv(channel_t *chan) { time_t now = time(NULL); - tor_assert(chan); + chan->timestamp_xfer_ms = monotime_coarse_absolute_msec(); chan->timestamp_active = now; chan->timestamp_recv = now; + + /* Clear any potential netflow padding timer. We're active */ + chan->next_padding_time_ms = 0; } /** @@ -4287,11 +4428,15 @@ void channel_timestamp_xmit(channel_t *chan) { time_t now = time(NULL); - tor_assert(chan); + chan->timestamp_xfer_ms = monotime_coarse_absolute_msec(); + chan->timestamp_active = now; chan->timestamp_xmit = now; + + /* Clear any potential netflow padding timer. We're active */ + chan->next_padding_time_ms = 0; } /*************************************************************** diff --git a/src/or/channel.h b/src/or/channel.h index 26aa93b5e2..ea280f2fd2 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,6 +11,8 @@ #include "or.h" #include "circuitmux.h" +#include "timers.h" +#include "handles.h" /* Channel handler function pointer typedefs */ typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *); @@ -22,6 +24,17 @@ TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s); typedef struct chan_cell_queue chan_cell_queue_t; /** + * This enum is used by channelpadding to decide when to pad channels. + * Don't add values to it without updating the checks in + * channelpadding_decide_to_pad_channel(). + */ +typedef enum { + CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS = 0, + CHANNEL_USED_FOR_FULL_CIRCS, + CHANNEL_USED_FOR_USER_TRAFFIC, +} channel_usage_info_t; + +/** * Channel struct; see the channel_t typedef in or.h. A channel is an * abstract interface for the OR-to-OR connection, similar to connection_or_t, * but without the strong coupling to the underlying TLS implementation. They @@ -34,11 +47,17 @@ struct channel_s { /** Magic number for type-checking cast macros */ uint32_t magic; + /** List entry for hashtable for global-identifier lookup. */ + HT_ENTRY(channel_s) gidmap_node; + + /** Handle entry for handle-based lookup */ + HANDLE_ENTRY(channel, channel_s); + /** Current channel state */ channel_state_t state; /** Globally unique ID number for a channel over the lifetime of a Tor - * process. + * process. This may not be 0. */ uint64_t global_identifier; @@ -48,6 +67,61 @@ struct channel_s { /** has this channel ever been open? */ unsigned int has_been_open:1; + /** + * This field indicates if the other side has enabled or disabled + * padding via either the link protocol version or + * channelpadding_negotiate cells. + * + * Clients can override this with ConnectionPadding in torrc to + * disable or force padding to relays, but relays cannot override the + * client's request. + */ + unsigned int padding_enabled:1; + + /** Cached value of our decision to pad (to avoid expensive + * checks during critical path statistics counting). */ + unsigned int currently_padding:1; + + /** Is there a pending netflow padding callback? */ + unsigned int pending_padding_callback:1; + + /** Is our peer likely to consider this channel canonical? */ + unsigned int is_canonical_to_peer:1; + + /** Has this channel ever been used for non-directory traffic? + * Used to decide what channels to pad, and when. */ + channel_usage_info_t channel_usage; + + /** When should we send a cell for netflow padding, in absolute + * milliseconds since monotime system start. 0 means no padding + * is scheduled. */ + uint64_t next_padding_time_ms; + + /** The callback pointer for the padding callbacks */ + tor_timer_t *padding_timer; + /** The handle to this channel (to free on canceled timers) */ + struct channel_handle_t *timer_handle; + + /** + * These two fields specify the minimum and maximum negotiated timeout + * values for inactivity (send or receive) before we decide to pad a + * channel. These fields can be set either via a PADDING_NEGOTIATE cell, + * or the torrc option ReducedConnectionPadding. The consensus parameters + * nf_ito_low and nf_ito_high are used to ensure that padding can only be + * negotiated to be less frequent than what is specified in the consensus. + * (This is done to prevent wingnut clients from requesting excessive + * padding). + * + * The actual timeout value is randomly chosen between these two values + * as per the table in channelpadding_get_netflow_inactive_timeout_ms(), + * after ensuring that these values do not specify lower timeouts than + * the consensus parameters. + * + * If these are 0, we have not negotiated or specified custom padding + * times, and instead use consensus defaults. */ + uint16_t padding_timeout_low_ms; + uint16_t padding_timeout_high_ms; + /** Why did we close? */ enum { @@ -87,6 +161,18 @@ struct channel_s { time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ + /** + * This is a high-resolution monotonic timestamp that marks when we + * believe the channel has actually sent or received data to/from + * the wire. Right now, it is used to determine when we should send + * a padding cell for channelpadding. + * + * XXX: Are we setting timestamp_xfer_ms in the right places to + * accurately reflect actual network data transfer? Or might this be + * very wrong wrt when bytes actually go on the wire? + */ + uint64_t timestamp_xfer_ms; + /* Methods implemented by the lower layer */ /** Free a channel */ @@ -214,8 +300,8 @@ struct channel_s { unsigned int is_bad_for_new_circs:1; /** True iff we have decided that the other end of this connection - * is a client. Channels with this flag set should never be used - * to satisfy an EXTEND request. */ + * is a client or bridge relay. Connections with this flag set should never + * be used to satisfy an EXTEND request. */ unsigned int is_client:1; /** Set if the channel was initiated remotely (came from a listener) */ @@ -516,9 +602,7 @@ channel_t * channel_get_for_extend(const char *rsa_id_digest, int *launch_out); /* Ask which of two channels is better for circuit-extension purposes */ -int channel_is_better(time_t now, - channel_t *a, channel_t *b, - int forgive_new_connections); +int channel_is_better(channel_t *a, channel_t *b); /** Channel lookups */ @@ -601,6 +685,7 @@ void channel_listener_dump_statistics(channel_listener_t *chan_l, int severity); void channel_listener_dump_transport_statistics(channel_listener_t *chan_l, int severity); +void channel_check_for_duplicates(void); void channel_update_bad_for_new_circs(const char *digest, int force); @@ -630,5 +715,8 @@ int packed_cell_is_destroy(channel_t *chan, const packed_cell_t *packed_cell, circid_t *circid_out); +/* Declare the handle helpers */ +HANDLE_DECL(channel, channel_s,) + #endif diff --git a/src/or/channelpadding.c b/src/or/channelpadding.c new file mode 100644 index 0000000000..f5248e8960 --- /dev/null +++ b/src/or/channelpadding.c @@ -0,0 +1,749 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2015, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of + * channelpadding_channel_to_channelinfo() */ +#define TOR_CHANNEL_INTERNAL_ + +#include "or.h" +#include "channel.h" +#include "channelpadding.h" +#include "channeltls.h" +#include "config.h" +#include "networkstatus.h" +#include "connection.h" +#include "connection_or.h" +#include "main.h" +#include "rephist.h" +#include "router.h" +#include "compat_time.h" +#include <event2/event.h> + +STATIC int channelpadding_get_netflow_inactive_timeout_ms(const channel_t *); +STATIC int channelpadding_send_disable_command(channel_t *); +STATIC int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *); + +/** The total number of pending channelpadding timers */ +static uint64_t total_timers_pending; + +/** These are cached consensus parameters for netflow */ +/** The timeout lower bound that is allowed before sending padding */ +static int consensus_nf_ito_low; +/** The timeout upper bound that is allowed before sending padding */ +static int consensus_nf_ito_high; +/** The timeout lower bound that is allowed before sending reduced padding */ +static int consensus_nf_ito_low_reduced; +/** The timeout upper bound that is allowed before sending reduced padding */ +static int consensus_nf_ito_high_reduced; +/** The connection timeout between relays */ +static int consensus_nf_conntimeout_relays; +/** The connection timeout for client connections */ +static int consensus_nf_conntimeout_clients; +/** Should we pad before circuits are actually used for client data? */ +static int consensus_nf_pad_before_usage; +/** Should we pad relay-to-relay connections? */ +static int consensus_nf_pad_relays; + +#define TOR_MSEC_PER_SEC 1000 +#define TOR_USEC_PER_MSEC 1000 + +/** + * How often do we get called by the connection housekeeping (ie: once + * per second) */ +#define TOR_HOUSEKEEPING_CALLBACK_MSEC 1000 +/** + * Additional extra time buffer on the housekeeping callback, since + * it can be delayed. This extra slack is used to decide if we should + * schedule a timer or wait for the next callback. */ +#define TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC 100 + +/** + * This macro tells us if either end of the channel is connected to a client. + * (If we're not a server, we're definitely a client. If the channel thinks + * its a client, use that. Then finally verify in the consensus). + */ +#define CHANNEL_IS_CLIENT(chan, options) \ + (!public_server_mode((options)) || (chan)->is_client || \ + !connection_or_digest_is_known_relay((chan)->identity_digest)) + +/** + * This function is called to update cached consensus parameters every time + * there is a consensus update. This allows us to move the consensus param + * search off of the critical path, so it does not need to be evaluated + * for every single connection, every second. + */ +void +channelpadding_new_consensus_params(networkstatus_t *ns) +{ +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW 1500 +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH 9500 +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN 0 +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX 60000 + consensus_nf_ito_low = networkstatus_get_param(ns, "nf_ito_low", + DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW, + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN, + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX); + consensus_nf_ito_high = networkstatus_get_param(ns, "nf_ito_high", + DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH, + consensus_nf_ito_low, + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX); + +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW 9000 +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH 14000 +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN 0 +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX 60000 + consensus_nf_ito_low_reduced = + networkstatus_get_param(ns, "nf_ito_low_reduced", + DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW, + DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN, + DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX); + + consensus_nf_ito_high_reduced = + networkstatus_get_param(ns, "nf_ito_high_reduced", + DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH, + consensus_nf_ito_low_reduced, + DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX); + +#define CONNTIMEOUT_RELAYS_DFLT (60*60) // 1 hour +#define CONNTIMEOUT_RELAYS_MIN 60 +#define CONNTIMEOUT_RELAYS_MAX (7*24*60*60) // 1 week + consensus_nf_conntimeout_relays = + networkstatus_get_param(ns, "nf_conntimeout_relays", + CONNTIMEOUT_RELAYS_DFLT, + CONNTIMEOUT_RELAYS_MIN, + CONNTIMEOUT_RELAYS_MAX); + +#define CIRCTIMEOUT_CLIENTS_DFLT (30*60) // 30 minutes +#define CIRCTIMEOUT_CLIENTS_MIN 60 +#define CIRCTIMEOUT_CLIENTS_MAX (24*60*60) // 24 hours + consensus_nf_conntimeout_clients = + networkstatus_get_param(ns, "nf_conntimeout_clients", + CIRCTIMEOUT_CLIENTS_DFLT, + CIRCTIMEOUT_CLIENTS_MIN, + CIRCTIMEOUT_CLIENTS_MAX); + + consensus_nf_pad_before_usage = + networkstatus_get_param(ns, "nf_pad_before_usage", 1, 0, 1); + + consensus_nf_pad_relays = + networkstatus_get_param(ns, "nf_pad_relays", 0, 0, 1); +} + +/** + * Get a random netflow inactive timeout keepalive period in milliseconds, + * the range for which is determined by consensus parameters, negotiation, + * configuration, or default values. The consensus parameters enforce the + * minimum possible value, to avoid excessively frequent padding. + * + * The ranges for this value were chosen to be low enough to ensure that + * routers do not emit a new netflow record for a connection due to it + * being idle. + * + * Specific timeout values for major routers are listed in Proposal 251. + * No major router appeared capable of setting an inactive timeout below 10 + * seconds, so we set the defaults below that value, since we can always + * scale back if it ends up being too much padding. + * + * Returns the next timeout period (in milliseconds) after which we should + * send a padding packet, or 0 if padding is disabled. + */ +STATIC int +channelpadding_get_netflow_inactive_timeout_ms(const channel_t *chan) +{ + int low_timeout = consensus_nf_ito_low; + int high_timeout = consensus_nf_ito_high; + int X1, X2; + + if (low_timeout == 0 && low_timeout == high_timeout) + return 0; // No padding + + /* If we have negotiated different timeout values, use those, but + * don't allow them to be lower than the consensus ones */ + if (chan->padding_timeout_low_ms && chan->padding_timeout_high_ms) { + low_timeout = MAX(low_timeout, chan->padding_timeout_low_ms); + high_timeout = MAX(high_timeout, chan->padding_timeout_high_ms); + } + + if (low_timeout == high_timeout) + return low_timeout; // No randomization + + /* + * This MAX() hack is here because we apply the timeout on both the client + * and the server. This creates the situation where the total time before + * sending a packet in either direction is actually + * min(client_timeout,server_timeout). + * + * If X is a random variable uniform from 0..R-1 (where R=high-low), + * then Y=max(X,X) has Prob(Y == i) = (2.0*i + 1)/(R*R). + * + * If we create a third random variable Z=min(Y,Y), then it turns out that + * Exp[Z] ~= Exp[X]. Here's a table: + * + * R Exp[X] Exp[Z] Exp[min(X,X)] Exp[max(X,X)] + * 2000 999.5 1066 666.2 1332.8 + * 3000 1499.5 1599.5 999.5 1999.5 + * 5000 2499.5 2666 1666.2 3332.8 + * 6000 2999.5 3199.5 1999.5 3999.5 + * 7000 3499.5 3732.8 2332.8 4666.2 + * 8000 3999.5 4266.2 2666.2 5332.8 + * 10000 4999.5 5328 3332.8 6666.2 + * 15000 7499.5 7995 4999.5 9999.5 + * 20000 9900.5 10661 6666.2 13332.8 + * + * In other words, this hack makes it so that when both the client and + * the guard are sending this padding, then the averages work out closer + * to the midpoint of the range, making the overhead easier to tune. + * If only one endpoint is padding (for example: if the relay does not + * support padding, but the client has set ConnectionPadding 1; or + * if the relay does support padding, but the client has set + * ReducedConnectionPadding 1), then the defense will still prevent + * record splitting, but with less overhead than the midpoint + * (as seen by the Exp[max(X,X)] column). + * + * To calculate average padding packet frequency (and thus overhead), + * index into the table by picking a row based on R = high-low. Then, + * use the appropriate column (Exp[Z] for two-sided padding, and + * Exp[max(X,X)] for one-sided padding). Finally, take this value + * and add it to the low timeout value. This value is the average + * frequency which padding packets will be sent. + */ + + X1 = crypto_rand_int(high_timeout - low_timeout); + X2 = crypto_rand_int(high_timeout - low_timeout); + return low_timeout + MAX(X1, X2); +} + +/** + * Update this channel's padding settings based on the PADDING_NEGOTIATE + * contents. + * + * Returns -1 on error; 1 on success. + */ +int +channelpadding_update_padding_for_channel(channel_t *chan, + const channelpadding_negotiate_t *pad_vars) +{ + if (pad_vars->version != 0) { + static ratelim_t version_limit = RATELIM_INIT(600); + + log_fn_ratelim(&version_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL, + "Got a PADDING_NEGOTIATE cell with an unknown version. Ignoring."); + return -1; + } + + // We should not allow malicious relays to disable or reduce padding for + // us as clients. In fact, we should only accept this cell at all if we're + // operating as a relay. Bridges should not accept it from relays, either + // (only from their clients). + if ((get_options()->BridgeRelay && + connection_or_digest_is_known_relay(chan->identity_digest)) || + !get_options()->ORPort_set) { + static ratelim_t relay_limit = RATELIM_INIT(600); + + log_fn_ratelim(&relay_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL, + "Got a PADDING_NEGOTIATE from relay at %s (%s). " + "This should not happen.", + chan->get_remote_descr(chan, 0), + hex_str(chan->identity_digest, DIGEST_LEN)); + return -1; + } + + chan->padding_enabled = (pad_vars->command == CHANNELPADDING_COMMAND_START); + + /* Min must not be lower than the current consensus parameter + nf_ito_low. */ + chan->padding_timeout_low_ms = MAX(consensus_nf_ito_low, + pad_vars->ito_low_ms); + + /* Max must not be lower than ito_low_ms */ + chan->padding_timeout_high_ms = MAX(chan->padding_timeout_low_ms, + pad_vars->ito_high_ms); + + log_fn(LOG_INFO,LD_OR, + "Negotiated padding=%d, lo=%d, hi=%d on "U64_FORMAT, + chan->padding_enabled, chan->padding_timeout_low_ms, + chan->padding_timeout_high_ms, + U64_PRINTF_ARG(chan->global_identifier)); + + return 1; +} + +/** + * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side not + * to send padding. + * + * Returns -1 on error, 0 on success. + */ +STATIC int +channelpadding_send_disable_command(channel_t *chan) +{ + channelpadding_negotiate_t disable; + cell_t cell; + + tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >= + MIN_LINK_PROTO_FOR_CHANNEL_PADDING); + + memset(&cell, 0, sizeof(cell_t)); + memset(&disable, 0, sizeof(channelpadding_negotiate_t)); + cell.command = CELL_PADDING_NEGOTIATE; + + channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP); + + if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, + &disable) < 0) + return -1; + + if (chan->write_cell(chan, &cell) == 1) + return 0; + else + return -1; +} + +/** + * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side to + * resume sending padding at some rate. + * + * Returns -1 on error, 0 on success. + */ +int +channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout, + uint16_t high_timeout) +{ + channelpadding_negotiate_t enable; + cell_t cell; + + tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >= + MIN_LINK_PROTO_FOR_CHANNEL_PADDING); + + memset(&cell, 0, sizeof(cell_t)); + memset(&enable, 0, sizeof(channelpadding_negotiate_t)); + cell.command = CELL_PADDING_NEGOTIATE; + + channelpadding_negotiate_set_command(&enable, CHANNELPADDING_COMMAND_START); + channelpadding_negotiate_set_ito_low_ms(&enable, low_timeout); + channelpadding_negotiate_set_ito_high_ms(&enable, high_timeout); + + if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, + &enable) < 0) + return -1; + + if (chan->write_cell(chan, &cell) == 1) + return 0; + else + return -1; +} + +/** + * Sends a CELL_PADDING cell on a channel if it has been idle since + * our callback was scheduled. + * + * This function also clears the pending padding timer and the callback + * flags. + */ +static void +channelpadding_send_padding_cell_for_callback(channel_t *chan) +{ + cell_t cell; + + /* Check that the channel is still valid and open */ + if (!chan || chan->state != CHANNEL_STATE_OPEN) { + if (chan) chan->pending_padding_callback = 0; + log_fn(LOG_INFO,LD_OR, + "Scheduled a netflow padding cell, but connection already closed."); + return; + } + + /* We should have a pending callback flag set. */ + if (BUG(chan->pending_padding_callback == 0)) + return; + + chan->pending_padding_callback = 0; + + if (!chan->next_padding_time_ms || + chan->has_queued_writes(chan)) { + /* We must have been active before the timer fired */ + chan->next_padding_time_ms = 0; + return; + } + + { + uint64_t now = monotime_coarse_absolute_msec(); + + log_fn(LOG_INFO,LD_OR, + "Sending netflow keepalive on "U64_FORMAT" to %s (%s) after " + I64_FORMAT" ms. Delta "I64_FORMAT"ms", + U64_PRINTF_ARG(chan->global_identifier), + safe_str_client(chan->get_remote_descr(chan, 0)), + safe_str_client(hex_str(chan->identity_digest, DIGEST_LEN)), + U64_PRINTF_ARG(now - chan->timestamp_xfer_ms), + U64_PRINTF_ARG(now - chan->next_padding_time_ms)); + } + + /* Clear the timer */ + chan->next_padding_time_ms = 0; + + /* Send the padding cell. This will cause the channel to get a + * fresh timestamp_active */ + memset(&cell, 0, sizeof(cell)); + cell.command = CELL_PADDING; + chan->write_cell(chan, &cell); +} + +/** + * tor_timer callback function for us to send padding on an idle channel. + * + * This function just obtains the channel from the callback handle, ensures + * it is still valid, and then hands it off to + * channelpadding_send_padding_cell_for_callback(), which checks if + * the channel is still idle before sending padding. + */ +static void +channelpadding_send_padding_callback(tor_timer_t *timer, void *args, + const struct monotime_t *when) +{ + channel_t *chan = channel_handle_get((struct channel_handle_t*)args); + (void)timer; (void)when; + + if (chan && CHANNEL_CAN_HANDLE_CELLS(chan)) { + /* Hrmm.. It might be nice to have an equivalent to assert_connection_ok + * for channels. Then we could get rid of the channeltls dependency */ + tor_assert(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn)->magic == + OR_CONNECTION_MAGIC); + assert_connection_ok(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), approx_time()); + + channelpadding_send_padding_cell_for_callback(chan); + } else { + log_fn(LOG_INFO,LD_OR, + "Channel closed while waiting for timer."); + } + + total_timers_pending--; +} + +/** + * Schedules a callback to send padding on a channel in_ms milliseconds from + * now. + * + * Returns CHANNELPADDING_WONTPAD on error, CHANNELPADDING_PADDING_SENT if we + * sent the packet immediately without a timer, and + * CHANNELPADDING_PADDING_SCHEDULED if we decided to schedule a timer. + */ +static channelpadding_decision_t +channelpadding_schedule_padding(channel_t *chan, int in_ms) +{ + struct timeval timeout; + tor_assert(!chan->pending_padding_callback); + + if (in_ms <= 0) { + chan->pending_padding_callback = 1; + channelpadding_send_padding_cell_for_callback(chan); + return CHANNELPADDING_PADDING_SENT; + } + + timeout.tv_sec = in_ms/TOR_MSEC_PER_SEC; + timeout.tv_usec = (in_ms%TOR_USEC_PER_MSEC)*TOR_USEC_PER_MSEC; + + if (!chan->timer_handle) { + chan->timer_handle = channel_handle_new(chan); + } + + if (chan->padding_timer) { + timer_set_cb(chan->padding_timer, + channelpadding_send_padding_callback, + chan->timer_handle); + } else { + chan->padding_timer = timer_new(channelpadding_send_padding_callback, + chan->timer_handle); + } + timer_schedule(chan->padding_timer, &timeout); + + rep_hist_padding_count_timers(++total_timers_pending); + + chan->pending_padding_callback = 1; + return CHANNELPADDING_PADDING_SCHEDULED; +} + +/** + * Calculates the number of milliseconds from now to schedule a padding cell. + * + * Returns the number of milliseconds from now (relative) to schedule the + * padding callback. If the padding timer is more than 1.1 seconds in the + * future, we return -1, to avoid scheduling excessive callbacks. If padding + * is disabled in the consensus, we return -2. + * + * Side-effects: Updates chan->next_padding_time_ms, storing an (absolute, not + * relative) millisecond representation of when we should send padding, unless + * other activity happens first. This side-effect allows us to avoid + * scheduling a libevent callback until we're within 1.1 seconds of the padding + * time. + */ +#define CHANNELPADDING_TIME_LATER -1 +#define CHANNELPADDING_TIME_DISABLED -2 +STATIC int64_t +channelpadding_compute_time_until_pad_for_netflow(channel_t *chan) +{ + uint64_t long_now = monotime_coarse_absolute_msec(); + + if (!chan->next_padding_time_ms) { + /* If the below line or crypto_rand_int() shows up on a profile, + * we can avoid getting a timeout until we're at least nf_ito_lo + * from a timeout window. That will prevent us from setting timers + * on connections that were active up to 1.5 seconds ago. + * Idle connections should only call this once every 5.5s on average + * though, so that might be a micro-optimization for little gain. */ + int64_t padding_timeout = + channelpadding_get_netflow_inactive_timeout_ms(chan); + + if (!padding_timeout) + return CHANNELPADDING_TIME_DISABLED; + + chan->next_padding_time_ms = padding_timeout + + chan->timestamp_xfer_ms; + } + + /* If the next padding time is beyond the maximum possible consensus value, + * then this indicates a clock jump, so just send padding now. This is + * better than using monotonic time because we want to avoid the situation + * where we wait around forever for monotonic time to move forward after + * a clock jump far into the past. + */ + if (chan->next_padding_time_ms > long_now + + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX) { + tor_fragile_assert(); + log_warn(LD_BUG, + "Channel padding timeout scheduled "I64_FORMAT"ms in the future. " + "Did the monotonic clock just jump?", + I64_PRINTF_ARG(chan->next_padding_time_ms - long_now)); + return 0; /* Clock jumped: Send padding now */ + } + + /* If the timeout will expire before the next time we're called (1000ms + from now, plus some slack), then calculate the number of milliseconds + from now which we should send padding, so we can schedule a callback + then. + */ + if (long_now + + (TOR_HOUSEKEEPING_CALLBACK_MSEC + TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC) + >= chan->next_padding_time_ms) { + int64_t ms_until_pad_for_netflow = chan->next_padding_time_ms - + long_now; + if (ms_until_pad_for_netflow < 0) { + log_warn(LD_BUG, + "Channel padding timeout scheduled "I64_FORMAT"ms in the past. " + "Did the monotonic clock just jump?", + I64_PRINTF_ARG(-ms_until_pad_for_netflow)); + return 0; /* Clock jumped: Send padding now */ + } + + return ms_until_pad_for_netflow; + } + return CHANNELPADDING_TIME_LATER; +} + +/** + * Returns a randomized value for channel idle timeout in seconds. + * The channel idle timeout governs how quickly we close a channel + * after its last circuit has disappeared. + * + * There are three classes of channels: + * 1. Client+non-canonical. These live for 3-4.5 minutes + * 2. relay to relay. These live for 45-75 min by default + * 3. Reduced padding clients. These live for 1.5-2.25 minutes. + * + * Also allows the default relay-to-relay value to be controlled by the + * consensus. + */ +unsigned int +channelpadding_get_channel_idle_timeout(const channel_t *chan, + int is_canonical) +{ + const or_options_t *options = get_options(); + unsigned int timeout; + + /* Non-canonical and client channels only last for 3-4.5 min when idle */ + if (!is_canonical || CHANNEL_IS_CLIENT(chan, options)) { +#define CONNTIMEOUT_CLIENTS_BASE 180 // 3 to 4.5 min + timeout = CONNTIMEOUT_CLIENTS_BASE + + crypto_rand_int(CONNTIMEOUT_CLIENTS_BASE/2); + } else { // Canonical relay-to-relay channels + // 45..75min or consensus +/- 25% + timeout = consensus_nf_conntimeout_relays; + timeout = 3*timeout/4 + crypto_rand_int(timeout/2); + } + + /* If ReducedConnectionPadding is set, we want to halve the duration of + * the channel idle timeout, since reducing the additional time that + * a channel stays open will reduce the total overhead for making + * new channels. This reduction in overhead/channel expense + * is important for mobile users. The option cannot be set by relays. + * + * We also don't reduce any values for timeout that the user explicitly + * set. + */ + if (options->ReducedConnectionPadding + && !options->CircuitsAvailableTimeout) { + timeout /= 2; + } + + return timeout; +} + +/** + * This function controls how long we keep idle circuits open, + * and how long we build predicted circuits. This behavior is under + * the control of channelpadding because circuit availability is the + * dominant factor in channel lifespan, which influences total padding + * overhead. + * + * Returns a randomized number of seconds in a range from + * CircuitsAvailableTimeout to 2*CircuitsAvailableTimeout. This value is halved + * if ReducedConnectionPadding is set. The default value of + * CircuitsAvailableTimeout can be controlled by the consensus. + */ +int +channelpadding_get_circuits_available_timeout(void) +{ + const or_options_t *options = get_options(); + int timeout = options->CircuitsAvailableTimeout; + + if (!timeout) { + timeout = consensus_nf_conntimeout_clients; + + /* If ReducedConnectionPadding is set, we want to halve the duration of + * the channel idle timeout, since reducing the additional time that + * a channel stays open will reduce the total overhead for making + * new connections. This reduction in overhead/connection expense + * is important for mobile users. The option cannot be set by relays. + * + * We also don't reduce any values for timeout that the user explicitly + * set. + */ + if (options->ReducedConnectionPadding) { + // half the value to 15..30min by default + timeout /= 2; + } + } + + // 30..60min by default + timeout = timeout + crypto_rand_int(timeout); + + return timeout; +} + +/** + * Calling this function on a channel causes it to tell the other side + * not to send padding, and disables sending padding from this side as well. + */ +void +channelpadding_disable_padding_on_channel(channel_t *chan) +{ + chan->padding_enabled = 0; + + // Send cell to disable padding on the other end + channelpadding_send_disable_command(chan); +} + +/** + * Calling this function on a channel causes it to tell the other side + * not to send padding, and reduces the rate that padding is sent from + * this side. + */ +void +channelpadding_reduce_padding_on_channel(channel_t *chan) +{ + /* Padding can be forced and reduced by clients, regardless of if + * the channel supports it. So we check for support here before + * sending any commands. */ + if (chan->padding_enabled) { + channelpadding_send_disable_command(chan); + } + + chan->padding_timeout_low_ms = consensus_nf_ito_low_reduced; + chan->padding_timeout_high_ms = consensus_nf_ito_high_reduced; + + log_fn(LOG_INFO,LD_OR, + "Reduced padding on channel "U64_FORMAT": lo=%d, hi=%d", + U64_PRINTF_ARG(chan->global_identifier), + chan->padding_timeout_low_ms, chan->padding_timeout_high_ms); +} + +/** + * This function is called once per second by run_connection_housekeeping(), + * but only if the channel is still open, valid, and non-wedged. + * + * It decides if and when we should send a padding cell, and if needed, + * schedules a callback to send that cell at the appropriate time. + * + * Returns an enum that represents the current padding decision state. + * Return value is currently used only by unit tests. + */ +channelpadding_decision_t +channelpadding_decide_to_pad_channel(channel_t *chan) +{ + const or_options_t *options = get_options(); + + /* Only pad open channels */ + if (chan->state != CHANNEL_STATE_OPEN) + return CHANNELPADDING_WONTPAD; + + if (chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS) { + if (!consensus_nf_pad_before_usage) + return CHANNELPADDING_WONTPAD; + } else if (chan->channel_usage != CHANNEL_USED_FOR_USER_TRAFFIC) { + return CHANNELPADDING_WONTPAD; + } + + if (chan->pending_padding_callback) + return CHANNELPADDING_PADDING_ALREADY_SCHEDULED; + + /* Don't pad the channel if we didn't negotiate it, but still + * allow clients to force padding if options->ChannelPadding is + * explicitly set to 1. + */ + if (!chan->padding_enabled && options->ConnectionPadding != 1) { + return CHANNELPADDING_WONTPAD; + } + + if (!chan->has_queued_writes(chan)) { + int is_client_channel = 0; + + if (CHANNEL_IS_CLIENT(chan, options)) { + is_client_channel = 1; + } + + /* If nf_pad_relays=1 is set in the consensus, we pad + * on *all* idle connections, relay-relay or relay-client. + * Otherwise pad only for client+bridge cons */ + if (is_client_channel || consensus_nf_pad_relays) { + int64_t pad_time_ms = + channelpadding_compute_time_until_pad_for_netflow(chan); + + if (pad_time_ms == CHANNELPADDING_TIME_DISABLED) { + return CHANNELPADDING_WONTPAD; + } else if (pad_time_ms == CHANNELPADDING_TIME_LATER) { + chan->currently_padding = 1; + return CHANNELPADDING_PADLATER; + } else { + if (BUG(pad_time_ms > INT_MAX)) { + pad_time_ms = INT_MAX; + } + /* We have to schedule a callback because we're called exactly once per + * second, but we don't want padding packets to go out exactly on an + * integer multiple of seconds. This callback will only be scheduled + * if we're within 1.1 seconds of the padding time. + */ + chan->currently_padding = 1; + return channelpadding_schedule_padding(chan, (int)pad_time_ms); + } + } else { + chan->currently_padding = 0; + return CHANNELPADDING_WONTPAD; + } + } else { + return CHANNELPADDING_PADLATER; + } +} + diff --git a/src/or/channelpadding.h b/src/or/channelpadding.h new file mode 100644 index 0000000000..2708ee9739 --- /dev/null +++ b/src/or/channelpadding.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2015, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ +#ifndef TOR_CHANNELPADDING_H +#define TOR_CHANNELPADDING_H + +#include "channelpadding_negotiation.h" + +typedef enum { + CHANNELPADDING_WONTPAD, + CHANNELPADDING_PADLATER, + CHANNELPADDING_PADDING_SCHEDULED, + CHANNELPADDING_PADDING_ALREADY_SCHEDULED, + CHANNELPADDING_PADDING_SENT, +} channelpadding_decision_t; + +channelpadding_decision_t channelpadding_decide_to_pad_channel(channel_t + *chan); +int channelpadding_update_padding_for_channel(channel_t *, + const channelpadding_negotiate_t + *chan); + +void channelpadding_disable_padding_on_channel(channel_t *chan); +void channelpadding_reduce_padding_on_channel(channel_t *chan); +int channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout, + uint16_t high_timeout); + +int channelpadding_get_circuits_available_timeout(void); +unsigned int channelpadding_get_channel_idle_timeout(const channel_t *, int); +void channelpadding_new_consensus_params(networkstatus_t *ns); + +#endif + diff --git a/src/or/channeltls.c b/src/or/channeltls.c index dbed95fb43..f44e4fc8ea 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -57,6 +57,9 @@ #include "routerlist.h" #include "scheduler.h" #include "torcert.h" +#include "networkstatus.h" +#include "channelpadding_negotiation.h" +#include "channelpadding.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -122,6 +125,8 @@ static void channel_tls_process_netinfo_cell(cell_t *cell, static int command_allowed_before_handshake(uint8_t command); static int enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *tlschan); +static void channel_tls_process_padding_negotiate_cell(cell_t *cell, + channel_tls_t *chan); /** * Do parts of channel_tls_t initialization common to channel_tls_connect() @@ -734,6 +739,15 @@ channel_tls_matches_target_method(channel_t *chan, return 0; } + /* real_addr is the address this connection came from. + * base_.addr is updated by connection_or_init_conn_from_address() + * to be the address in the descriptor. It may be tempting to + * allow either address to be allowed, but if we did so, it would + * enable someone who steals a relay's keys to impersonate/MITM it + * from anywhere on the Internet! (Because they could make long-lived + * TLS connections from anywhere to all relays, and wait for them to + * be used for extends). + */ return tor_addr_eq(&(tlschan->conn->real_addr), target); } @@ -1098,9 +1112,16 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) /* We note that we're on the internet whenever we read a cell. This is * a fast operation. */ entry_guards_note_internet_connectivity(get_guard_selection_info()); + rep_hist_padding_count_read(PADDING_TYPE_TOTAL); + + if (TLS_CHAN_TO_BASE(chan)->currently_padding) + rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL); switch (cell->command) { case CELL_PADDING: + rep_hist_padding_count_read(PADDING_TYPE_CELL); + if (TLS_CHAN_TO_BASE(chan)->currently_padding) + rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL); ++stats_n_padding_cells_processed; /* do nothing */ break; @@ -1111,6 +1132,10 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) ++stats_n_netinfo_cells_processed; PROCESS_CELL(netinfo, cell, chan); break; + case CELL_PADDING_NEGOTIATE: + ++stats_n_netinfo_cells_processed; + PROCESS_CELL(padding_negotiate, cell, chan); + break; case CELL_CREATE: case CELL_CREATE_FAST: case CELL_CREATED: @@ -1566,9 +1591,12 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) /* We set this after sending the verions cell. */ /*XXXXX symbolic const.*/ - chan->base_.wide_circ_ids = + TLS_CHAN_TO_BASE(chan)->wide_circ_ids = chan->conn->link_proto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS; - chan->conn->wide_circ_ids = chan->base_.wide_circ_ids; + chan->conn->wide_circ_ids = TLS_CHAN_TO_BASE(chan)->wide_circ_ids; + + TLS_CHAN_TO_BASE(chan)->padding_enabled = + chan->conn->link_proto >= MIN_LINK_PROTO_FOR_CHANNEL_PADDING; if (send_certs) { if (connection_or_send_certs_cell(chan->conn) < 0) { @@ -1595,6 +1623,43 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) } /** + * Process a 'padding_negotiate' cell + * + * This function is called to handle an incoming PADDING_NEGOTIATE cell; + * enable or disable padding accordingly, and read and act on its timeout + * value contents. + */ +static void +channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) +{ + channelpadding_negotiate_t *negotiation; + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + + if (chan->conn->link_proto < MIN_LINK_PROTO_FOR_CHANNEL_PADDING) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a PADDING_NEGOTIATE cell on v%d connection; dropping.", + chan->conn->link_proto); + return; + } + + if (channelpadding_negotiate_parse(&negotiation, cell->payload, + CELL_PAYLOAD_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received malformed PADDING_NEGOTIATE cell on v%d connection; " + "dropping.", chan->conn->link_proto); + + return; + } + + channelpadding_update_padding_for_channel(TLS_CHAN_TO_BASE(chan), + negotiation); + + channelpadding_negotiate_free(negotiation); +} + +/** * Process a 'netinfo' cell * * This function is called to handle an incoming NETINFO cell; read and act @@ -1611,6 +1676,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) const uint8_t *cp, *end; uint8_t n_other_addrs; time_t now = time(NULL); + const routerinfo_t *me = router_get_my_routerinfo(); long apparent_skew = 0; tor_addr_t my_apparent_addr = TOR_ADDR_NULL; @@ -1654,6 +1720,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) tor_assert(tor_mem_is_zero( (const char*)(chan->conn->handshake_state-> authenticated_ed25519_peer_id.pubkey), 32)); + /* If the client never authenticated, it's a tor client or bridge + * relay, and we must not use it for EXTEND requests (nor could we, as + * there are no authenticated peer IDs) */ + channel_mark_client(TLS_CHAN_TO_BASE(chan)); channel_set_circid_type(TLS_CHAN_TO_BASE(chan), NULL, chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS); @@ -1689,8 +1759,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + + if (!get_options()->BridgeRelay && me && + get_uint32(my_addr_ptr) == htonl(me->addr)) { + TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; + } + } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); + + if (!get_options()->BridgeRelay && me && + !tor_addr_is_null(&me->ipv6_addr) && + tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { + TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; + } } n_other_addrs = (uint8_t) *cp++; @@ -1706,6 +1788,14 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) connection_or_close_for_error(chan->conn, 0); return; } + /* A relay can connect from anywhere and be canonical, so + * long as it tells you from where it came. This may be a bit + * concerning.. Luckily we have another check in + * channel_tls_matches_target_method() to ensure that extends + * only go to the IP they ask for. + * + * XXX: Bleh. That check is not used if the connection is canonical. + */ if (tor_addr_eq(&addr, &(chan->conn->real_addr))) { connection_or_set_canonical(chan->conn, 1); break; @@ -1714,6 +1804,21 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) --n_other_addrs; } + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && + channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { + const char *descr = + TLS_CHAN_TO_BASE(chan)->get_remote_descr(TLS_CHAN_TO_BASE(chan), 0); + log_info(LD_OR, + "We made a connection to a relay at %s (fp=%s) but we think " + "they will not consider this connection canonical. They " + "think we are at %s, but we think its %s.", + safe_str(descr), + safe_str(hex_str(chan->conn->identity_digest, DIGEST_LEN)), + safe_str(tor_addr_is_null(&my_apparent_addr) ? + "<none>" : fmt_and_decorate_addr(&my_apparent_addr)), + safe_str(fmt_addr32(me->addr))); + } + /* Act on apparent skew. */ /** Warn when we get a netinfo skew with at least this value. */ #define NETINFO_NOTICE_SKEW 3600 diff --git a/src/or/channeltls.h b/src/or/channeltls.h index 729e595615..1f9a39d8a4 100644 --- a/src/or/channeltls.h +++ b/src/or/channeltls.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c index cdcb6deae4..4c0bd9e455 100644 --- a/src/or/circpathbias.c +++ b/src/or/circpathbias.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h index ce76689d5f..2a4c316807 100644 --- a/src/or/circpathbias.h +++ b/src/or/circpathbias.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 79962e8dbb..16cef0e56b 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -73,7 +73,6 @@ static int circuit_deliver_create_cell(circuit_t *circ, static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit); static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); static int onion_extend_cpath(origin_circuit_t *circ); -static int count_acceptable_nodes(smartlist_t *routers); static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); /** This function tries to get a channel to the specified endpoint, @@ -817,12 +816,7 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ) * creating on behalf of others. */ return 0; } - if (options->FastFirstHopPK == -1) { - /* option is "auto", so look at the consensus. */ - return networkstatus_get_param(NULL, "usecreatefast", 1, 0, 1); - } - - return options->FastFirstHopPK; + return networkstatus_get_param(NULL, "usecreatefast", 0, 0, 1); } /** Return true if <b>circ</b> is the type of circuit we want to count @@ -940,9 +934,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) memset(&cc, 0, sizeof(cc)); if (circ->build_state->onehop_tunnel) control_event_bootstrap(BOOTSTRAP_STATUS_ONEHOP_CREATE, 0); - else + else { control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0); + /* If this is not a one-hop tunnel, the channel is being used + * for traffic that wants anonymity and protection from traffic + * analysis (such as netflow record retention). That means we want + * to pad it. + */ + if (circ->base_.n_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS) + circ->base_.n_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS; + } + node = node_get_by_id(circ->base_.n_chan->identity_digest); fast = should_use_create_fast_for_circuit(circ); if (!fast) { @@ -1546,13 +1549,98 @@ onionskin_answer(or_circuit_t *circ, return 0; } -/** Choose a length for a circuit of purpose <b>purpose</b>: three + the - * number of endpoints that would give something away about our destination. +/** Helper for new_route_len(). Choose a circuit length for purpose + * <b>purpose</b>: DEFAULT_ROUTE_LEN (+ 1 if someone else chose the + * exit). If someone else chose the exit, they could be colluding + * with the exit, so add a randomly selected node to preserve + * anonymity. + * + * Here, "exit node" sometimes means an OR acting as an internal + * endpoint, rather than as a relay to an external endpoint. This + * means there need to be at least DEFAULT_ROUTE_LEN routers between + * us and the internal endpoint to preserve the same anonymity + * properties that we would get when connecting to an external + * endpoint. These internal endpoints can include: + * + * - Connections to a directory of hidden services + * (CIRCUIT_PURPOSE_C_GENERAL) + * + * - A client connecting to an introduction point, which the hidden + * service picked (CIRCUIT_PURPOSE_C_INTRODUCING, via + * circuit_get_open_circ_or_launch() which rewrites it from + * CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) + * + * - A hidden service connecting to a rendezvous point, which the + * client picked (CIRCUIT_PURPOSE_S_CONNECT_REND, via + * rend_service_receive_introduction() and + * rend_service_relaunch_rendezvous) + * + * There are currently two situations where we picked the exit node + * ourselves, making DEFAULT_ROUTE_LEN a safe circuit length: + * + * - We are a hidden service connecting to an introduction point + * (CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, via + * rend_service_launch_establish_intro()) + * + * - We are a router testing its own reachabiity + * (CIRCUIT_PURPOSE_TESTING, via consider_testing_reachability()) + * + * onion_pick_cpath_exit() bypasses us (by not calling + * new_route_len()) in the one-hop tunnel case, so we don't need to + * handle that. + */ +static int +route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) +{ + int routelen = DEFAULT_ROUTE_LEN; + int known_purpose = 0; + + if (!exit_ei) + return routelen; + + switch (purpose) { + /* These two purposes connect to a router that we chose, so + * DEFAULT_ROUTE_LEN is safe. */ + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: + /* hidden service connecting to introduction point */ + case CIRCUIT_PURPOSE_TESTING: + /* router reachability testing */ + known_purpose = 1; + break; + + /* These three purposes connect to a router that someone else + * might have chosen, so add an extra hop to protect anonymity. */ + case CIRCUIT_PURPOSE_C_GENERAL: + /* connecting to hidden service directory */ + case CIRCUIT_PURPOSE_C_INTRODUCING: + /* client connecting to introduction point */ + case CIRCUIT_PURPOSE_S_CONNECT_REND: + /* hidden service connecting to rendezvous point */ + known_purpose = 1; + routelen++; + break; + + default: + /* Got a purpose not listed above along with a chosen exit. + * Increase the circuit length by one anyway for safety. */ + routelen++; + break; + } + + if (BUG(exit_ei && !known_purpose)) { + log_warn(LD_BUG, "Unhandled purpose %d with a chosen exit; " + "assuming routelen %d.", purpose, routelen); + } + return routelen; +} + +/** Choose a length for a circuit of purpose <b>purpose</b> and check + * if enough routers are available. * * If the routerlist <b>nodes</b> doesn't have enough routers * to handle the desired path length, return -1. */ -static int +STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) { int num_acceptable_routers; @@ -1560,11 +1648,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) tor_assert(nodes); - routelen = DEFAULT_ROUTE_LEN; - if (exit_ei && - purpose != CIRCUIT_PURPOSE_TESTING && - purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) - routelen++; + routelen = route_len_for_purpose(purpose, exit_ei); num_acceptable_routers = count_acceptable_nodes(nodes); @@ -1748,15 +1832,16 @@ choose_good_exit_server_general(int need_uptime, int need_capacity) * we'll retry later in this function with need_update and * need_capacity set to 0. */ } - if (!(node->is_valid || options->AllowInvalid_ & ALLOW_INVALID_EXIT)) { + if (!(node->is_valid)) { /* if it's invalid and we don't want it */ n_supported[i] = -1; // log_fn(LOG_DEBUG,"Skipping node %s (index %d) -- invalid router.", // router->nickname, i); continue; /* skip invalid routers */ } - if (options->ExcludeSingleHopRelays && - node_allows_single_hop_exits(node)) { + /* We do not allow relays that allow single hop exits by default. Option + * was deprecated in 0.2.9.2-alpha and removed in 0.3.1.0-alpha. */ + if (node_allows_single_hop_exits(node)) { n_supported[i] = -1; continue; } @@ -1888,7 +1973,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags, const or_options_t *options) { const node_t *rp_node = NULL; - const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; const int need_desc = (flags & CRN_NEED_DESC) != 0; const int pref_addr = (flags & CRN_PREF_ADDR) != 0; const int direct_conn = (flags & CRN_DIRECT_CONN) != 0; @@ -1900,7 +1984,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags, /* Add all running nodes to all_live_nodes */ router_add_running_nodes_to_smartlist(all_live_nodes, - allow_invalid, 0, 0, 0, need_desc, pref_addr, @@ -1942,9 +2025,6 @@ pick_rendezvous_node(router_crn_flags_t flags) { const or_options_t *options = get_options(); - if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS) - flags |= CRN_ALLOW_INVALID; - #ifdef ENABLE_TOR2WEB_MODE /* We want to connect directly to the node if we can */ router_crn_flags_t direct_flags = flags; @@ -2001,8 +2081,6 @@ choose_good_exit_server(uint8_t purpose, switch (purpose) { case CIRCUIT_PURPOSE_C_GENERAL: - if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE) - flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ return router_choose_random_node(NULL, options->ExcludeNodes, flags); else @@ -2188,8 +2266,8 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) /** Return the number of routers in <b>routers</b> that are currently up * and available for building circuits through. */ -static int -count_acceptable_nodes(smartlist_t *nodes) +MOCK_IMPL(STATIC int, +count_acceptable_nodes, (smartlist_t *nodes)) { int num=0; @@ -2200,10 +2278,6 @@ count_acceptable_nodes(smartlist_t *nodes) if (! node->is_running) // log_debug(LD_CIRC,"Nope, the directory says %d is not running.",i); continue; - /* XXX This clause makes us count incorrectly: if AllowInvalidRouters - * allows this node in some places, then we're getting an inaccurate - * count. For now, be conservative and don't count it. But later we - * should try to be smarter. */ if (! node->is_valid) // log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i); continue; @@ -2274,8 +2348,6 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_UPTIME; if (state->need_capacity) flags |= CRN_NEED_CAPACITY; - if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE) - flags |= CRN_ALLOW_INVALID; choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; @@ -2328,8 +2400,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state, if (state->need_capacity) flags |= CRN_NEED_CAPACITY; } - if (options->AllowInvalid_ & ALLOW_INVALID_ENTRY) - flags |= CRN_ALLOW_INVALID; choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index ddb070b427..45d9b2fb75 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -77,6 +77,9 @@ void circuit_upgrade_circuits_from_guard_wait(void); #ifdef CIRCUITBUILD_PRIVATE STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); +STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, + smartlist_t *nodes); +MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); #if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags, const or_options_t *options); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 54a7db9dbf..86b0aa097a 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1,7 +1,7 @@ /* Copyright 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -78,6 +78,7 @@ #include "rephist.h" #include "routerlist.h" #include "routerset.h" +#include "channelpadding.h" #include "ht.h" @@ -358,8 +359,8 @@ channel_note_destroy_pending(channel_t *chan, circid_t id) /** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with * circuit ID <b>id</b> -- typically, because it has been sent. */ -MOCK_IMPL(void, channel_note_destroy_not_pending, - (channel_t *chan, circid_t id)) +MOCK_IMPL(void, +channel_note_destroy_not_pending,(channel_t *chan, circid_t id)) { circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); if (circ) { @@ -814,6 +815,11 @@ init_circuit_base(circuit_t *circ) circ->global_circuitlist_idx = smartlist_len(circuit_get_global_list()) - 1; } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idle circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (1*60) + /** Allocate space for a new circuit, initializing with <b>p_circ_id</b> * and <b>p_conn</b>. Add it to the global circuit list. */ @@ -841,6 +847,41 @@ origin_circuit_new(void) circuit_build_times_update_last_circ(get_circuit_build_times_mutable()); + if (! circuit_build_times_disabled(get_options()) && + circuit_build_times_needs_circuits(get_circuit_build_times())) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + circ->circuit_idle_timeout = IDLE_TIMEOUT_WHILE_LEARNING; + } else { + // This should always be larger than the current port prediction time + // remaining, or else we'll end up with the case where a circuit times out + // and another one is built, effectively doubling the timeout window. + // + // We also randomize it by up to 5% more (ie 5% of 0 to 3600 seconds, + // depending on how much circuit prediction time is remaining) so that + // we don't close a bunch of unused circuits all at the same time. + int prediction_time_remaining = + predicted_ports_prediction_time_remaining(time(NULL)); + circ->circuit_idle_timeout = prediction_time_remaining+1+ + crypto_rand_int(1+prediction_time_remaining/20); + + if (circ->circuit_idle_timeout <= 0) { + log_warn(LD_BUG, + "Circuit chose a negative idle timeout of %d based on " + "%d seconds of predictive building remaining.", + circ->circuit_idle_timeout, + prediction_time_remaining); + circ->circuit_idle_timeout = IDLE_TIMEOUT_WHILE_LEARNING; + } + + log_info(LD_CIRC, + "Circuit " U64_FORMAT " chose an idle timeout of %d based on " + "%d seconds of predictive building remaining.", + U64_PRINTF_ARG(circ->global_identifier), + circ->circuit_idle_timeout, + prediction_time_remaining); + } + return circ; } @@ -943,10 +984,6 @@ circuit_free(circuit_t *circ) crypto_cipher_free(ocirc->n_crypto); crypto_digest_free(ocirc->n_digest); - if (ocirc->hs_token) { - hs_circuitmap_remove_circuit(ocirc); - } - if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; tor_assert(other->base_.magic == OR_CIRCUIT_MAGIC); @@ -978,6 +1015,11 @@ circuit_free(circuit_t *circ) /* Remove from map. */ circuit_set_n_circid_chan(circ, 0, NULL); + /* Clear HS circuitmap token from this circ (if any) */ + if (circ->hs_token) { + hs_circuitmap_remove_circuit(circ); + } + /* Clear cell queue _after_ removing it from the map. Otherwise our * "active" checks will be violated. */ cell_queue_clear(&circ->n_chan_cells); @@ -1989,10 +2031,10 @@ single_conn_free_bytes(connection_t *conn) } if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); - if (dir_conn->zlib_state) { - result += tor_zlib_state_size(dir_conn->zlib_state); - tor_zlib_free(dir_conn->zlib_state); - dir_conn->zlib_state = NULL; + if (dir_conn->compress_state) { + result += tor_compress_state_size(dir_conn->compress_state); + tor_compress_free(dir_conn->compress_state); + dir_conn->compress_state = NULL; } } return result; diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index 6abee37dc4..d647062f46 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index 96a3647aab..ee0d5c718b 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h index 00745ac4a1..42a46aaa47 100644 --- a/src/or/circuitmux.h +++ b/src/or/circuitmux.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c index 0219459cdb..c2440b13f0 100644 --- a/src/or/circuitmux_ewma.c +++ b/src/or/circuitmux_ewma.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index a7b8961ac6..1f04408789 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index 6e73372550..51d580a1a4 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h index c748f82d5e..8a1dec4bfd 100644 --- a/src/or/circuitstats.h +++ b/src/or/circuitstats.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuituse.c b/src/or/circuituse.c index c2b450606b..9f9d3abf7c 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -705,18 +705,15 @@ circuit_expire_building(void) } } - /* If this is a hidden service client circuit which is far enough - * along in connecting to its destination, and we haven't already - * flagged it as 'timed out', and the user has not told us to - * close such circs immediately on timeout, flag it as 'timed out' - * so we'll launch another intro or rend circ, but don't mark it - * for close yet. + /* If this is a hidden service client circuit which is far enough along in + * connecting to its destination, and we haven't already flagged it as + * 'timed out', flag it so we'll launch another intro or rend circ, but + * don't mark it for close yet. * * (Circs flagged as 'timed out' are given a much longer timeout * period above, so we won't close them in the next call to * circuit_expire_building.) */ - if (!(options->CloseHSClientCircuitsImmediatelyOnTimeout) && - !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) { + if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) { switch (victim->purpose) { case CIRCUIT_PURPOSE_C_REND_READY: /* We only want to spare a rend circ if it has been specified in @@ -750,8 +747,7 @@ circuit_expire_building(void) /* If this is a service-side rendezvous circuit which is far * enough along in connecting to its destination, consider sparing * it. */ - if (!(options->CloseHSServiceRendCircuitsImmediatelyOnTimeout) && - !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) && + if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) && victim->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { log_info(LD_CIRC,"Marking circ %u (state %d:%s, purpose %d) " "as timed-out HS circ; relaunching rendezvous attempt.", @@ -1240,8 +1236,8 @@ circuit_predict_and_launch_new(void) /** Build a new test circuit every 5 minutes */ #define TESTING_CIRCUIT_INTERVAL 300 -/** This function is called once a second, if router_have_min_dir_info() is - * true. Its job is to make sure all services we offer have enough circuits +/** This function is called once a second, if router_have_minimum_dir_info() + * is true. Its job is to make sure all services we offer have enough circuits * available. Some services just want enough circuits for current tasks, * whereas others want a minimum set of idle circuits hanging around. */ @@ -1383,11 +1379,6 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } -/** If we haven't yet decided on a good timeout value for circuit - * building, we close idles circuits aggressively so we can get more - * data points. */ -#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) - /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ @@ -1397,21 +1388,15 @@ circuit_expire_old_circuits_clientside(void) struct timeval cutoff, now; tor_gettimeofday(&now); - cutoff = now; last_expired_clientside_circuits = now.tv_sec; - if (! circuit_build_times_disabled(get_options()) && - circuit_build_times_needs_circuits(get_circuit_build_times())) { - /* Circuits should be shorter lived if we need more of them - * for learning a good build timeout */ - cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; - } else { - cutoff.tv_sec -= get_options()->CircuitIdleTimeout; - } - SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; + + cutoff = now; + cutoff.tv_sec -= TO_ORIGIN_CIRCUIT(circ)->circuit_idle_timeout; + /* If the circuit has been dirty for too long, and there are no streams * on it, mark it for close. */ @@ -1437,8 +1422,10 @@ circuit_expire_old_circuits_clientside(void) (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) || circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { - log_debug(LD_CIRC, - "Closing circuit that has been unused for %ld msec.", + log_info(LD_CIRC, + "Closing circuit "U64_FORMAT + " that has been unused for %ld msec.", + U64_PRINTF_ARG(TO_ORIGIN_CIRCUIT(circ)->global_identifier), tv_mdiff(&circ->timestamp_began, &now)); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { diff --git a/src/or/circuituse.h b/src/or/circuituse.h index e5f8700ea2..ad4c214a3b 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/command.c b/src/or/command.c index 5866c386e4..c667cbbe52 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -326,10 +326,19 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } + if (connection_or_digest_is_known_relay(chan->identity_digest)) { + rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); + // Needed for chutney: Sometimes relays aren't in the consensus yet, and + // get marked as clients. This resets their channels once they appear. + // Probably useful for normal operation wrt relay flapping, too. + chan->is_client = 0; + } else { + channel_mark_client(chan); + } + if (create_cell->handshake_type != ONION_HANDSHAKE_TYPE_FAST) { /* hand it off to the cpuworkers, and then return. */ - if (connection_or_digest_is_known_relay(chan->identity_digest)) - rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); + if (assign_onionskin_to_cpuworker(circ, create_cell) < 0) { log_debug(LD_GENERAL,"Failed to hand off onionskin. Closing."); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); @@ -344,8 +353,14 @@ command_process_create_cell(cell_t *cell, channel_t *chan) int len; created_cell_t created_cell; - /* Make sure we never try to use the OR connection on which we - * received this cell to satisfy an EXTEND request, */ + /* If the client used CREATE_FAST, it's probably a tor client or bridge + * relay, and we must not use it for EXTEND requests (in most cases, we + * won't have an authenticated peer ID for the extend). + * Public relays on 0.2.9 and later will use CREATE_FAST if they have no + * ntor onion key for this relay, but that should be a rare occurrence. + * Clients on 0.3.1 and later avoid using CREATE_FAST as much as they can, + * even during bootstrap, so the CREATE_FAST check is most accurate for + * earlier tor client versions. */ channel_mark_client(chan); memset(&created_cell, 0, sizeof(created_cell)); diff --git a/src/or/command.h b/src/or/command.h index 12cda6a463..5079d42e75 100644 --- a/src/or/command.h +++ b/src/or/command.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/config.c b/src/or/config.c index 9edc6b799b..7d2ebbdd03 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -69,10 +69,12 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" #include "connection_or.h" +#include "consdiffmgr.h" #include "control.h" #include "confparse.h" #include "cpuworker.h" @@ -99,7 +101,6 @@ #include "statefile.h" #include "transports.h" #include "ext_orport.h" -#include "torgzip.h" #ifdef _WIN32 #include <shlobj.h> #endif @@ -205,10 +206,10 @@ static config_var_t option_vars_[] = { V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), V(AllowDotExit, BOOL, "0"), - V(AllowInvalidNodes, CSV, "middle,rendezvous"), + OBSOLETE("AllowInvalidNodes"), V(AllowNonRFC953Hostnames, BOOL, "0"), - V(AllowSingleHopCircuits, BOOL, "0"), - V(AllowSingleHopExits, BOOL, "0"), + OBSOLETE("AllowSingleHopCircuits"), + OBSOLETE("AllowSingleHopExits"), V(AlternateBridgeAuthority, LINELIST, NULL), V(AlternateDirAuthority, LINELIST, NULL), OBSOLETE("AlternateHSAuthority"), @@ -242,9 +243,11 @@ static config_var_t option_vars_[] = { V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), V(CellStatistics, BOOL, "0"), + V(PaddingStatistics, BOOL, "1"), V(LearnCircuitBuildTimeout, BOOL, "1"), V(CircuitBuildTimeout, INTERVAL, "0"), - V(CircuitIdleTimeout, INTERVAL, "1 hour"), + OBSOLETE("CircuitIdleTimeout"), + V(CircuitsAvailableTimeout, INTERVAL, "0"), V(CircuitStreamTimeout, INTERVAL, "0"), V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), @@ -261,7 +264,7 @@ static config_var_t option_vars_[] = { V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), V(ContactInfo, STRING, NULL), - V(ControlListenAddress, LINELIST, NULL), + OBSOLETE("ControlListenAddress"), VPORT(ControlPort), V(ControlPortFileGroupReadable,BOOL, "0"), V(ControlPortWriteToFile, FILENAME, NULL), @@ -278,7 +281,7 @@ static config_var_t option_vars_[] = { V(DisableNetwork, BOOL, "0"), V(DirAllowPrivateAddresses, BOOL, "0"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"), - V(DirListenAddress, LINELIST, NULL), + OBSOLETE("DirListenAddress"), V(DirPolicy, LINELIST, NULL), VPORT(DirPort), V(DirPortFrontPage, FILENAME, NULL), @@ -292,7 +295,7 @@ static config_var_t option_vars_[] = { OBSOLETE("DisableV2DirectoryInfo_"), OBSOLETE("DynamicDHGroups"), VPORT(DNSPort), - V(DNSListenAddress, LINELIST, NULL), + OBSOLETE("DNSListenAddress"), V(DownloadExtraInfo, BOOL, "0"), V(TestingEnableConnBwEvent, BOOL, "0"), V(TestingEnableCellStatsEvent, BOOL, "0"), @@ -303,7 +306,7 @@ static config_var_t option_vars_[] = { V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), - V(ExcludeSingleHopRelays, BOOL, "1"), + OBSOLETE("ExcludeSingleHopRelays"), V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), @@ -323,7 +326,7 @@ static config_var_t option_vars_[] = { OBSOLETE("FallbackNetworkstatusFile"), V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), - V(FastFirstHopPK, AUTOBOOL, "auto"), + OBSOLETE("FastFirstHopPK"), V(FetchDirInfoEarly, BOOL, "0"), V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), @@ -360,8 +363,8 @@ static config_var_t option_vars_[] = { VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"), V(HidServAuth, LINELIST, NULL), - V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"), - V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"), + OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"), + OBSOLETE("CloseHSServiceRendCircuitsImmediatelyOnTimeout"), V(HiddenServiceSingleHopMode, BOOL, "0"), V(HiddenServiceNonAnonymousMode,BOOL, "0"), V(HTTPProxy, STRING, NULL), @@ -395,20 +398,20 @@ static config_var_t option_vars_[] = { V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), - V(MyFamily, STRING, NULL), + VAR("MyFamily", LINELIST, MyFamily_lines, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), OBSOLETE("NamingAuthoritativeDirectory"), - V(NATDListenAddress, LINELIST, NULL), + OBSOLETE("NATDListenAddress"), VPORT(NATDPort), V(Nickname, STRING, NULL), - V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), - V(WarnUnsafeSocks, BOOL, "1"), + OBSOLETE("PredictedPortsRelevanceTime"), + OBSOLETE("WarnUnsafeSocks"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), V(NumCPUs, UINT, "0"), V(NumDirectoryGuards, UINT, "0"), V(NumEntryGuards, UINT, "0"), V(OfflineMasterKey, BOOL, "0"), - V(ORListenAddress, LINELIST, NULL), + OBSOLETE("ORListenAddress"), VPORT(ORPort), V(OutboundBindAddress, LINELIST, NULL), V(OutboundBindAddressOR, LINELIST, NULL), @@ -458,6 +461,8 @@ static config_var_t option_vars_[] = { V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), V(RecommendedPackages, LINELIST, NULL), + V(ReducedConnectionPadding, BOOL, "0"), + V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), @@ -481,7 +486,7 @@ static config_var_t option_vars_[] = { V(SchedulerHighWaterMark__, MEMUNIT, "101 MB"), V(SchedulerMaxFlushCells__, UINT, "1000"), V(ShutdownWaitLength, INTERVAL, "30 seconds"), - V(SocksListenAddress, LINELIST, NULL), + OBSOLETE("SocksListenAddress"), V(SocksPolicy, LINELIST, NULL), VPORT(SocksPort), V(SocksTimeout, INTERVAL, "2 minutes"), @@ -494,10 +499,10 @@ static config_var_t option_vars_[] = { V(TokenBucketRefillInterval, MSEC_INTERVAL, "100 msec"), V(Tor2webMode, BOOL, "0"), V(Tor2webRendezvousPoints, ROUTERSET, NULL), - V(TLSECGroup, STRING, NULL), + OBSOLETE("TLSECGroup"), V(TrackHostExits, CSV, NULL), V(TrackHostExitsExpire, INTERVAL, "30 minutes"), - V(TransListenAddress, LINELIST, NULL), + OBSOLETE("TransListenAddress"), VPORT(TransPort), V(TransProxyType, STRING, "default"), OBSOLETE("TunnelDirConns"), @@ -553,11 +558,13 @@ static config_var_t option_vars_[] = { "10800, 21600, 43200"), /* With the ClientBootstrapConsensus*Download* below: * Clients with only authorities will try: - * - 3 authorities over 10 seconds, then wait 60 minutes. + * - at least 3 authorities over 10 seconds, then exponentially backoff, + * with the next attempt 3-21 seconds later, * Clients with authorities and fallbacks will try: - * - 2 authorities and 4 fallbacks over 21 seconds, then wait 60 minutes. + * - at least 2 authorities and 4 fallbacks over 21 seconds, then + * exponentially backoff, with the next attempts 4-33 seconds later, * Clients will also retry when an application request arrives. - * After a number of failed reqests, clients retry every 3 days + 1 hour. + * After a number of failed requests, clients retry every 3 days + 1 hour. * * Clients used to try 2 authorities over 10 seconds, then wait for * 60 minutes or an application request. @@ -662,35 +669,8 @@ static const config_deprecation_t option_deprecation_notes_[] = { /* Deprecated since 0.2.9.2-alpha... */ { "AllowDotExit", "Unrestricted use of the .exit notation can be used for " "a wide variety of application-level attacks." }, - { "AllowInvalidNodes", "There is no reason to enable this option; at best " - "it will make you easier to track." }, - { "AllowSingleHopCircuits", "Almost no relays actually allow single-hop " - "exits, making this option pointless." }, - { "AllowSingleHopExits", "Turning this on will make your relay easier " - "to abuse." }, { "ClientDNSRejectInternalAddresses", "Turning this on makes your client " "easier to fingerprint, and may open you to esoteric attacks." }, - { "ExcludeSingleHopRelays", "Turning it on makes your client easier to " - "fingerprint." }, - { "FastFirstHopPK", "Changing this option does not make your client more " - "secure, but does make it easier to fingerprint." }, - { "CloseHSClientCircuitsImmediatelyOnTimeout", "This option makes your " - "client easier to fingerprint." }, - { "CloseHSServiceRendCircuitsImmediatelyOnTimeout", "This option makes " - "your hidden services easier to fingerprint." }, - { "WarnUnsafeSocks", "Changing this option makes it easier for you " - "to accidentally lose your anonymity by leaking DNS information" }, - { "TLSECGroup", "The default is a nice secure choice; the other option " - "is less secure." }, - { "ControlListenAddress", "Use ControlPort instead." }, - { "DirListenAddress", "Use DirPort instead, possibly with the " - "NoAdvertise sub-option" }, - { "DNSListenAddress", "Use DNSPort instead." }, - { "SocksListenAddress", "Use SocksPort instead." }, - { "TransListenAddress", "Use TransPort instead." }, - { "NATDListenAddress", "Use NATDPort instead." }, - { "ORListenAddress", "Use ORPort instead, possibly with the " - "NoAdvertise sub-option" }, /* End of options deprecated since 0.2.9.2-alpha. */ { NULL, NULL } @@ -707,7 +687,9 @@ static int options_transition_affects_workers( const or_options_t *old_options, const or_options_t *new_options); static int options_transition_affects_descriptor( const or_options_t *old_options, const or_options_t *new_options); -static int check_nickname_list(char **lst, const char *name, char **msg); +static int normalize_nickname_list(config_line_t **normalized_out, + const config_line_t *lst, const char *name, + char **msg); static char *get_bindaddr_from_transport_listen_line(const char *line, const char *transport); static int parse_ports(or_options_t *options, int validate_only, @@ -915,6 +897,7 @@ or_options_free(or_options_t *options) tor_free(options->BridgePassword_AuthDigest_); tor_free(options->command_arg); tor_free(options->master_key_fname); + config_free_lines(options->MyFamily); config_free(&options_format, options); } @@ -1553,23 +1536,6 @@ get_effective_bwburst(const or_options_t *options) return (uint32_t)bw; } -/** Return True if any changes from <b>old_options</b> to - * <b>new_options</b> needs us to refresh our TLS context. */ -static int -options_transition_requires_fresh_tls_context(const or_options_t *old_options, - const or_options_t *new_options) -{ - tor_assert(new_options); - - if (!old_options) - return 0; - - if (!opt_streq(old_options->TLSECGroup, new_options->TLSECGroup)) - return 1; - - return 0; -} - /** * Return true if changing the configuration from <b>old</b> to <b>new</b> * affects the guard susbsystem. @@ -1788,13 +1754,6 @@ options_act(const or_options_t *old_options) log_warn(LD_BUG,"Error initializing keys; exiting"); return -1; } - } else if (old_options && - options_transition_requires_fresh_tls_context(old_options, - options)) { - if (router_initialize_tls_context() < 0) { - log_warn(LD_BUG,"Error initializing TLS context."); - return -1; - } } /* Write our PID to the PID file. If we do not have write permissions we @@ -1815,6 +1774,15 @@ options_act(const or_options_t *old_options) return -1; } + if (server_mode(options)) { + static int cdm_initialized = 0; + if (cdm_initialized == 0) { + cdm_initialized = 1; + consdiffmgr_configure(NULL); + consdiffmgr_validate(); + } + } + if (init_control_cookie_authentication(options->CookieAuthentication) < 0) { log_warn(LD_CONFIG,"Error creating control cookie authentication file."); return -1; @@ -2347,7 +2315,7 @@ print_usage(void) printf( "Copyright (c) 2001-2004, Roger Dingledine\n" "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n" -"Copyright (c) 2007-2016, The Tor Project, Inc.\n\n" +"Copyright (c) 2007-2017, The Tor Project, Inc.\n\n" "tor -f <torrc> [args]\n" "See man page for options, or https://www.torproject.org/ for " "documentation.\n"); @@ -2809,13 +2777,13 @@ compute_publishserverdescriptor(or_options_t *options) #define MIN_REND_POST_PERIOD (10*60) #define MIN_REND_POST_PERIOD_TESTING (5) -/** Highest allowable value for PredictedPortsRelevanceTime; if this is - * too high, our selection of exits will decrease for an extended - * period of time to an uncomfortable level .*/ -#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) +/** Highest allowable value for CircuitsAvailableTimeout. + * If this is too large, client connections will stay open for too long, + * incurring extra padding overhead. */ +#define MAX_CIRCS_AVAILABLE_TIME (24*60*60) /** Highest allowable value for RendPostPeriod. */ -#define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) +#define MAX_DIR_PERIOD ((7*24*60*60)/2) /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ @@ -2998,6 +2966,10 @@ options_validate(or_options_t *old_options, or_options_t *options, tor_assert(msg); *msg = NULL; + if (parse_ports(options, 1, msg, &n_ports, + &world_writable_control_socket) < 0) + return -1; + /* Set UseEntryGuards from the configured value, before we check it below. * We change UseEntryGuards when it's incompatible with other options, * but leave UseEntryGuards_option with the original value. @@ -3016,10 +2988,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "for details.", uname); } - if (parse_ports(options, 1, msg, &n_ports, - &world_writable_control_socket) < 0) - return -1; - if (parse_outbound_addresses(options, 1, msg) < 0) return -1; @@ -3112,14 +3080,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (strcasecmp(options->TransProxyType, "default") && !options->TransPort_set) { - REJECT("Cannot use TransProxyType without any valid TransPort or " - "TransListenAddress."); + REJECT("Cannot use TransProxyType without any valid TransPort."); } } #else if (options->TransPort_set) - REJECT("TransPort and TransListenAddress are disabled " - "in this build."); + REJECT("TransPort is disabled in this build."); #endif if (options->TokenBucketRefillInterval <= 0 @@ -3156,15 +3122,6 @@ options_validate(or_options_t *old_options, or_options_t *options, } } - if (options->TLSECGroup && (strcasecmp(options->TLSECGroup, "P256") && - strcasecmp(options->TLSECGroup, "P224"))) { - COMPLAIN("Unrecognized TLSECGroup: Falling back to the default."); - tor_free(options->TLSECGroup); - } - if (!evaluate_ecgroup_for_tls(options->TLSECGroup)) { - REJECT("Unsupported TLSECGroup."); - } - if (options->ExcludeNodes && options->StrictNodes) { COMPLAIN("You have asked to exclude certain relays from all positions " "in your circuits. Expect hidden services and other Tor " @@ -3372,28 +3329,6 @@ options_validate(or_options_t *old_options, or_options_t *options, server_mode(options)); options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3; - options->AllowInvalid_ = 0; - - if (options->AllowInvalidNodes) { - SMARTLIST_FOREACH_BEGIN(options->AllowInvalidNodes, const char *, cp) { - if (!strcasecmp(cp, "entry")) - options->AllowInvalid_ |= ALLOW_INVALID_ENTRY; - else if (!strcasecmp(cp, "exit")) - options->AllowInvalid_ |= ALLOW_INVALID_EXIT; - else if (!strcasecmp(cp, "middle")) - options->AllowInvalid_ |= ALLOW_INVALID_MIDDLE; - else if (!strcasecmp(cp, "introduction")) - options->AllowInvalid_ |= ALLOW_INVALID_INTRODUCTION; - else if (!strcasecmp(cp, "rendezvous")) - options->AllowInvalid_ |= ALLOW_INVALID_RENDEZVOUS; - else { - tor_asprintf(msg, - "Unrecognized value '%s' in AllowInvalidNodes", cp); - return -1; - } - } SMARTLIST_FOREACH_END(cp); - } - if (!options->SafeLogging || !strcasecmp(options->SafeLogging, "0")) { options->SafeLogging_ = SAFELOG_SCRUB_NONE; @@ -3429,6 +3364,14 @@ options_validate(or_options_t *old_options, or_options_t *options, options->DirPort_set = 0; } + if (server_mode(options) && options->ConnectionPadding != -1) { + REJECT("Relays must use 'auto' for the ConnectionPadding setting."); + } + + if (server_mode(options) && options->ReducedConnectionPadding != 0) { + REJECT("Relays cannot set ReducedConnectionPadding. "); + } + if (options->MinUptimeHidServDirectoryV2 < 0) { log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at " "least 0 seconds. Changing to 0."); @@ -3450,17 +3393,17 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->PredictedPortsRelevanceTime > - MAX_PREDICTED_CIRCS_RELEVANCE) { - log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; " - "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE); - options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; - } - /* Check the Single Onion Service options */ if (options_validate_single_onion(options, msg) < 0) return -1; + if (options->CircuitsAvailableTimeout > MAX_CIRCS_AVAILABLE_TIME) { + // options_t is immutable for new code (the above code is older), + // so just make the user fix the value themselves rather than + // silently keep a shadow value lower than what they asked for. + REJECT("CircuitsAvailableTimeout is too large. Max is 24 hours."); + } + #ifdef ENABLE_TOR2WEB_MODE if (options->Tor2webMode && options->UseEntryGuards) { /* tor2web mode clients do not (and should not) use entry guards @@ -3510,6 +3453,20 @@ options_validate(or_options_t *old_options, or_options_t *options, return -1; } + /* Inform the hidden service operator that pinning EntryNodes can possibly + * be harmful for the service anonymity. */ + if (options->EntryNodes && + routerset_is_list(options->EntryNodes) && + (options->RendConfigLines != NULL)) { + log_warn(LD_CONFIG, + "EntryNodes is set with multiple entries and at least one " + "hidden service is configured. Pinning entry nodes can possibly " + "be harmful to the service anonymity. Because of this, we " + "recommend you either don't do that or make sure you know what " + "you are doing. For more details, please look at " + "https://trac.torproject.org/projects/tor/ticket/21155."); + } + /* Single Onion Services: non-anonymous hidden services */ if (rend_service_non_anonymous_mode_enabled(options)) { log_warn(LD_CONFIG, @@ -3865,13 +3822,14 @@ options_validate(or_options_t *old_options, or_options_t *options, "have it group-readable."); } - if (options->MyFamily && options->BridgeRelay) { + if (options->MyFamily_lines && options->BridgeRelay) { log_warn(LD_CONFIG, "Listing a family for a bridge relay is not " "supported: it can reveal bridge fingerprints to censors. " "You should also make sure you aren't listing this bridge's " "fingerprint in any other MyFamily."); } - if (check_nickname_list(&options->MyFamily, "MyFamily", msg)) + if (normalize_nickname_list(&options->MyFamily, + options->MyFamily_lines, "MyFamily", msg)) return -1; for (cl = options->NodeFamilies; cl; cl = cl->next) { routerset_t *rs = routerset_new(); @@ -4068,13 +4026,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "AlternateDirAuthority and AlternateBridgeAuthority configured."); } - if (options->AllowSingleHopExits && !options->DirAuthorities) { - COMPLAIN("You have set AllowSingleHopExits; now your relay will allow " - "others to make one-hop exits. However, since by default most " - "clients avoid relays that set this option, most clients will " - "ignore you."); - } - #define CHECK_DEFAULT(arg) \ STMT_BEGIN \ if (!options->TestingTorNetwork && \ @@ -4579,7 +4530,7 @@ options_transition_affects_descriptor(const or_options_t *old_options, get_effective_bwburst(old_options) != get_effective_bwburst(new_options) || !opt_streq(old_options->ContactInfo, new_options->ContactInfo) || - !opt_streq(old_options->MyFamily, new_options->MyFamily) || + !config_lines_eq(old_options->MyFamily, new_options->MyFamily) || !opt_streq(old_options->AccountingStart, new_options->AccountingStart) || old_options->AccountingMax != new_options->AccountingMax || old_options->AccountingRule != new_options->AccountingRule || @@ -4675,27 +4626,36 @@ get_default_conf_file(int defaults_file) #endif } -/** Verify whether lst is a string containing valid-looking comma-separated - * nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' to any nickname - * or fingerprint that needs it. Return 0 on success. +/** Verify whether lst is a list of strings containing valid-looking + * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' + * to any nickname or fingerprint that needs it. Also splits comma-separated + * list elements into multiple elements. Return 0 on success. * Warn and return -1 on failure. */ static int -check_nickname_list(char **lst, const char *name, char **msg) +normalize_nickname_list(config_line_t **normalized_out, + const config_line_t *lst, const char *name, + char **msg) { - int r = 0; - smartlist_t *sl; - int changes = 0; - - if (!*lst) + if (!lst) return 0; - sl = smartlist_new(); - smartlist_split_string(sl, *lst, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); + config_line_t *new_nicknames = NULL; + config_line_t **new_nicknames_next = &new_nicknames; - SMARTLIST_FOREACH_BEGIN(sl, char *, s) + const config_line_t *cl; + for (cl = lst; cl; cl = cl->next) { + const char *line = cl->value; + if (!line) + continue; + + int valid_line = 1; + smartlist_t *sl = smartlist_new(); + smartlist_split_string(sl, line, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); + SMARTLIST_FOREACH_BEGIN(sl, char *, s) { + char *normalized = NULL; if (!is_legal_nickname_or_hexdigest(s)) { // check if first char is dollar if (s[0] != '$') { @@ -4704,36 +4664,45 @@ check_nickname_list(char **lst, const char *name, char **msg) tor_asprintf(&prepended, "$%s", s); if (is_legal_nickname_or_hexdigest(prepended)) { - // The nickname is valid when it's prepended, swap the current - // version with a prepended one - tor_free(s); - SMARTLIST_REPLACE_CURRENT(sl, s, prepended); - changes = 1; - continue; + // The nickname is valid when it's prepended, set it as the + // normalized version + normalized = prepended; + } else { + // Still not valid, free and fallback to error message + tor_free(prepended); } - - // Still not valid, free and fallback to error message - tor_free(prepended); } - tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); - r = -1; - break; + if (!normalized) { + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); + valid_line = 0; + break; + } + } else { + normalized = tor_strdup(s); } - } - SMARTLIST_FOREACH_END(s); - // Replace the caller's nickname list with a fixed one - if (changes && r == 0) { - char *newNicknames = smartlist_join_strings(sl, ", ", 0, NULL); - tor_free(*lst); - *lst = newNicknames; + config_line_t *next = tor_malloc_zero(sizeof(*next)); + next->key = tor_strdup(cl->key); + next->value = normalized; + next->next = NULL; + + *new_nicknames_next = next; + new_nicknames_next = &next->next; + } SMARTLIST_FOREACH_END(s); + + SMARTLIST_FOREACH(sl, char *, s, tor_free(s)); + smartlist_free(sl); + + if (!valid_line) { + config_free_lines(new_nicknames); + return -1; + } } - SMARTLIST_FOREACH(sl, char *, s, tor_free(s)); - smartlist_free(sl); + *normalized_out = new_nicknames; - return r; + return 0; } /** Learn config file name from command line arguments, or use the default. @@ -4935,9 +4904,21 @@ options_init_from_torrc(int argc, char **argv) printf("OpenSSL \t\t%-15s\t\t%s\n", crypto_openssl_get_header_version_str(), crypto_openssl_get_version_str()); - printf("Zlib \t\t%-15s\t\t%s\n", - tor_zlib_get_header_version_str(), - tor_zlib_get_version_str()); + if (tor_compress_supports_method(ZLIB_METHOD)) { + printf("Zlib \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZLIB_METHOD), + tor_compress_header_version_str(ZLIB_METHOD)); + } + if (tor_compress_supports_method(LZMA_METHOD)) { + printf("Liblzma \t\t%-15s\t\t%s\n", + tor_compress_version_str(LZMA_METHOD), + tor_compress_header_version_str(LZMA_METHOD)); + } + if (tor_compress_supports_method(ZSTD_METHOD)) { + printf("Libzstd \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZSTD_METHOD), + tor_compress_header_version_str(ZSTD_METHOD)); + } //TODO: Hex versions? exit(0); } @@ -5077,6 +5058,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, config_line_t *cl; int retval; setopt_err_t err = SETOPT_ERR_MISC; + int cf_has_include = 0; tor_assert(msg); oldoptions = global_options; /* get_options unfortunately asserts if @@ -5093,7 +5075,8 @@ options_init_from_string(const char *cf_defaults, const char *cf, if (!body) continue; /* get config lines, assign them */ - retval = config_get_lines(body, &cl, 1); + retval = config_get_lines_include(body, &cl, 1, + body == cf ? &cf_has_include : NULL); if (retval < 0) { err = SETOPT_ERR_PARSE; goto err; @@ -5121,6 +5104,8 @@ options_init_from_string(const char *cf_defaults, const char *cf, goto err; } + newoptions->IncludeUsed = cf_has_include; + /* If this is a testing network configuration, change defaults * for a list of dependent config options, re-initialize newoptions * with the new defaults, and assign all options to it second time. */ @@ -5164,7 +5149,8 @@ options_init_from_string(const char *cf_defaults, const char *cf, if (!body) continue; /* get config lines, assign them */ - retval = config_get_lines(body, &cl, 1); + retval = config_get_lines_include(body, &cl, 1, + body == cf ? &cf_has_include : NULL); if (retval < 0) { err = SETOPT_ERR_PARSE; goto err; @@ -5187,6 +5173,8 @@ options_init_from_string(const char *cf_defaults, const char *cf, } } + newoptions->IncludeUsed = cf_has_include; + /* Validate newoptions */ if (options_validate(oldoptions, newoptions, newdefaultoptions, 0, msg) < 0) { @@ -6423,14 +6411,9 @@ warn_client_dns_cache(const char *option, int disabling) /** * Parse port configuration for a single port type. * - * Read entries of the "FooPort" type from the list <b>ports</b>, and - * entries of the "FooListenAddress" type from the list - * <b>listenaddrs</b>. Two syntaxes are supported: a legacy syntax - * where FooPort is at most a single entry containing a port number and - * where FooListenAddress has any number of address:port combinations; - * and a new syntax where there are no FooListenAddress entries and - * where FooPort can have any number of entries of the format - * "[Address:][Port] IsolationOptions". + * Read entries of the "FooPort" type from the list <b>ports</b>. Syntax is + * that FooPort can have any number of entries of the format + * "[Address:][Port] IsolationOptions". * * In log messages, describe the port type as <b>portname</b>. * @@ -6444,9 +6427,6 @@ warn_client_dns_cache(const char *option, int disabling) * ports are not on a local address. If CL_PORT_FORBID_NONLOCAL is set, * this is a control port with no password set: don't even allow it. * - * Unless CL_PORT_ALLOW_EXTRA_LISTENADDR is set in <b>flags</b>, warn - * if FooListenAddress is set but FooPort is 0. - * * If CL_PORT_SERVER_OPTIONS is set in <b>flags</b>, do not allow stream * isolation options in the FooPort entries; instead allow the * server-port option set. @@ -6461,7 +6441,6 @@ warn_client_dns_cache(const char *option, int disabling) STATIC int parse_port_config(smartlist_t *out, const config_line_t *ports, - const config_line_t *listenaddrs, const char *portname, int listener_type, const char *defaultaddr, @@ -6478,90 +6457,12 @@ parse_port_config(smartlist_t *out, const unsigned forbid_nonlocal = flags & CL_PORT_FORBID_NONLOCAL; const unsigned default_to_group_writable = flags & CL_PORT_DFLT_GROUP_WRITABLE; - const unsigned allow_spurious_listenaddr = - flags & CL_PORT_ALLOW_EXTRA_LISTENADDR; const unsigned takes_hostnames = flags & CL_PORT_TAKES_HOSTNAMES; const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET; int got_zero_port=0, got_nonzero_port=0; char *unix_socket_path = NULL; - /* FooListenAddress is deprecated; let's make it work like it used to work, - * though. */ - if (listenaddrs) { - int mainport = defaultport; - - if (ports && ports->next) { - log_warn(LD_CONFIG, "%sListenAddress can't be used when there are " - "multiple %sPort lines", portname, portname); - return -1; - } else if (ports) { - if (!strcmp(ports->value, "auto")) { - mainport = CFG_AUTO_PORT; - } else { - int ok; - mainport = (int)tor_parse_long(ports->value, 10, 0, 65535, &ok, NULL); - if (!ok) { - log_warn(LD_CONFIG, "%sListenAddress can only be used with a single " - "%sPort with value \"auto\" or 1-65535 and no options set.", - portname, portname); - return -1; - } - } - } - - if (mainport == 0) { - if (allow_spurious_listenaddr) - return 1; /*DOCDOC*/ - log_warn(LD_CONFIG, "%sPort must be defined if %sListenAddress is used", - portname, portname); - return -1; - } - - if (use_server_options && out) { - /* Add a no_listen port. */ - port_cfg_t *cfg = port_cfg_new(0); - cfg->type = listener_type; - cfg->port = mainport; - tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */ - cfg->server_cfg.no_listen = 1; - cfg->server_cfg.bind_ipv4_only = 1; - /* cfg->entry_cfg defaults are already set by port_cfg_new */ - smartlist_add(out, cfg); - } - - for (; listenaddrs; listenaddrs = listenaddrs->next) { - tor_addr_t addr; - uint16_t port = 0; - if (tor_addr_port_lookup(listenaddrs->value, &addr, &port) < 0) { - log_warn(LD_CONFIG, "Unable to parse %sListenAddress '%s'", - portname, listenaddrs->value); - return -1; - } - if (out) { - port_cfg_t *cfg = port_cfg_new(0); - cfg->type = listener_type; - cfg->port = port ? port : mainport; - tor_addr_copy(&cfg->addr, &addr); - cfg->entry_cfg.session_group = SESSION_GROUP_UNSET; - cfg->entry_cfg.isolation_flags = ISO_DEFAULT; - cfg->server_cfg.no_advertise = 1; - smartlist_add(out, cfg); - } - } - - if (warn_nonlocal && out) { - if (is_control) - warn_nonlocal_controller_ports(out, forbid_nonlocal); - else if (is_ext_orport) - warn_nonlocal_ext_orports(out, portname); - else - warn_nonlocal_client_ports(out, portname, listener_type); - } - return 0; - } /* end if (listenaddrs) */ - - /* No ListenAddress lines. If there's no FooPort, then maybe make a default - * one. */ + /* If there's no FooPort, then maybe make a default one. */ if (! ports) { if (defaultport && defaultaddr && out) { port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0); @@ -7032,36 +6933,35 @@ parse_ports(or_options_t *options, int validate_only, const unsigned gw_flag = options->SocksSocketsGroupWritable ? CL_PORT_DFLT_GROUP_WRITABLE : 0; if (parse_port_config(ports, - options->SocksPort_lines, options->SocksListenAddress, + options->SocksPort_lines, "Socks", CONN_TYPE_AP_LISTENER, "127.0.0.1", 9050, - CL_PORT_WARN_NONLOCAL|CL_PORT_ALLOW_EXTRA_LISTENADDR| - CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) { - *msg = tor_strdup("Invalid SocksPort/SocksListenAddress configuration"); + CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) { + *msg = tor_strdup("Invalid SocksPort configuration"); goto err; } if (parse_port_config(ports, - options->DNSPort_lines, options->DNSListenAddress, + options->DNSPort_lines, "DNS", CONN_TYPE_AP_DNS_LISTENER, "127.0.0.1", 0, CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES) < 0) { - *msg = tor_strdup("Invalid DNSPort/DNSListenAddress configuration"); + *msg = tor_strdup("Invalid DNSPort configuration"); goto err; } if (parse_port_config(ports, - options->TransPort_lines, options->TransListenAddress, + options->TransPort_lines, "Trans", CONN_TYPE_AP_TRANS_LISTENER, "127.0.0.1", 0, CL_PORT_WARN_NONLOCAL) < 0) { - *msg = tor_strdup("Invalid TransPort/TransListenAddress configuration"); + *msg = tor_strdup("Invalid TransPort configuration"); goto err; } if (parse_port_config(ports, - options->NATDPort_lines, options->NATDListenAddress, + options->NATDPort_lines, "NATD", CONN_TYPE_AP_NATD_LISTENER, "127.0.0.1", 0, CL_PORT_WARN_NONLOCAL) < 0) { - *msg = tor_strdup("Invalid NatdPort/NatdListenAddress configuration"); + *msg = tor_strdup("Invalid NatdPort configuration"); goto err; } { @@ -7077,16 +6977,14 @@ parse_ports(or_options_t *options, int validate_only, if (parse_port_config(ports, options->ControlPort_lines, - options->ControlListenAddress, "Control", CONN_TYPE_CONTROL_LISTENER, "127.0.0.1", 0, control_port_flags) < 0) { - *msg = tor_strdup("Invalid ControlPort/ControlListenAddress " - "configuration"); + *msg = tor_strdup("Invalid ControlPort configuration"); goto err; } - if (parse_port_config(ports, options->ControlSocket, NULL, + if (parse_port_config(ports, options->ControlSocket, "ControlSocket", CONN_TYPE_CONTROL_LISTENER, NULL, 0, control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) { @@ -7096,15 +6994,15 @@ parse_ports(or_options_t *options, int validate_only, } if (! options->ClientOnly) { if (parse_port_config(ports, - options->ORPort_lines, options->ORListenAddress, + options->ORPort_lines, "OR", CONN_TYPE_OR_LISTENER, "0.0.0.0", 0, CL_PORT_SERVER_OPTIONS) < 0) { - *msg = tor_strdup("Invalid ORPort/ORListenAddress configuration"); + *msg = tor_strdup("Invalid ORPort configuration"); goto err; } if (parse_port_config(ports, - options->ExtORPort_lines, NULL, + options->ExtORPort_lines, "ExtOR", CONN_TYPE_EXT_OR_LISTENER, "127.0.0.1", 0, CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) { @@ -7112,11 +7010,11 @@ parse_ports(or_options_t *options, int validate_only, goto err; } if (parse_port_config(ports, - options->DirPort_lines, options->DirListenAddress, + options->DirPort_lines, "Dir", CONN_TYPE_DIR_LISTENER, "0.0.0.0", 0, CL_PORT_SERVER_OPTIONS) < 0) { - *msg = tor_strdup("Invalid DirPort/DirListenAddress configuration"); + *msg = tor_strdup("Invalid DirPort configuration"); goto err; } } diff --git a/src/or/config.h b/src/or/config.h index 6645532514..27aec7fe3d 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -157,7 +157,7 @@ smartlist_t *get_options_for_server_transport(const char *transport); #define CL_PORT_NO_STREAM_OPTIONS (1u<<0) #define CL_PORT_WARN_NONLOCAL (1u<<1) -#define CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) +/* Was CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) */ #define CL_PORT_SERVER_OPTIONS (1u<<3) #define CL_PORT_FORBID_NONLOCAL (1u<<4) #define CL_PORT_TAKES_HOSTNAMES (1u<<5) @@ -193,7 +193,6 @@ STATIC int have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, char **msg); STATIC int parse_port_config(smartlist_t *out, const config_line_t *ports, - const config_line_t *listenaddrs, const char *portname, int listener_type, const char *defaultaddr, diff --git a/src/or/confparse.c b/src/or/confparse.c index 9b13a91856..abae7e33dc 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -2,7 +2,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -31,8 +31,6 @@ static int config_parse_msec_interval(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void config_reset(const config_format_t *fmt, void *options, const config_var_t *var, int use_defaults); -static config_line_t *config_lines_dup_and_filter(const config_line_t *inp, - const char *key); /** Allocate an empty configuration object of a given format type. */ void * @@ -80,120 +78,6 @@ config_expand_abbrev(const config_format_t *fmt, const char *option, return option; } -/** Helper: allocate a new configuration option mapping 'key' to 'val', - * append it to *<b>lst</b>. */ -void -config_line_append(config_line_t **lst, - const char *key, - const char *val) -{ - config_line_t *newline; - - newline = tor_malloc_zero(sizeof(config_line_t)); - newline->key = tor_strdup(key); - newline->value = tor_strdup(val); - newline->next = NULL; - while (*lst) - lst = &((*lst)->next); - - (*lst) = newline; -} - -/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL - * if no such key exists. For handling commandline-only options only; other - * options should be looked up in the appropriate data structure. */ -const config_line_t * -config_line_find(const config_line_t *lines, - const char *key) -{ - const config_line_t *cl; - for (cl = lines; cl; cl = cl->next) { - if (!strcmp(cl->key, key)) - return cl; - } - return NULL; -} - -/** Helper: parse the config string and strdup into key/value - * strings. Set *result to the list, or NULL if parsing the string - * failed. Return 0 on success, -1 on failure. Warn and ignore any - * misformatted lines. - * - * If <b>extended</b> is set, then treat keys beginning with / and with + as - * indicating "clear" and "append" respectively. */ -int -config_get_lines(const char *string, config_line_t **result, int extended) -{ - config_line_t *list = NULL, **next; - char *k, *v; - const char *parse_err; - - next = &list; - do { - k = v = NULL; - string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err); - if (!string) { - log_warn(LD_CONFIG, "Error while parsing configuration: %s", - parse_err?parse_err:"<unknown>"); - config_free_lines(list); - tor_free(k); - tor_free(v); - return -1; - } - if (k && v) { - unsigned command = CONFIG_LINE_NORMAL; - if (extended) { - if (k[0] == '+') { - char *k_new = tor_strdup(k+1); - tor_free(k); - k = k_new; - command = CONFIG_LINE_APPEND; - } else if (k[0] == '/') { - char *k_new = tor_strdup(k+1); - tor_free(k); - k = k_new; - tor_free(v); - v = tor_strdup(""); - command = CONFIG_LINE_CLEAR; - } - } - /* This list can get long, so we keep a pointer to the end of it - * rather than using config_line_append over and over and getting - * n^2 performance. */ - *next = tor_malloc_zero(sizeof(config_line_t)); - (*next)->key = k; - (*next)->value = v; - (*next)->next = NULL; - (*next)->command = command; - next = &((*next)->next); - } else { - tor_free(k); - tor_free(v); - } - } while (*string); - - *result = list; - return 0; -} - -/** - * Free all the configuration lines on the linked list <b>front</b>. - */ -void -config_free_lines(config_line_t *front) -{ - config_line_t *tmp; - - while (front) { - tmp = front; - front = tmp->next; - - tor_free(tmp->key); - tor_free(tmp->value); - tor_free(tmp); - } -} - /** If <b>key</b> is a deprecated configuration option, return the message * explaining why it is deprecated (which may be an empty string). Return NULL * if it is not deprecated. The <b>key</b> field must be fully expanded. */ @@ -633,36 +517,6 @@ config_value_needs_escape(const char *value) return 0; } -/** Return a newly allocated deep copy of the lines in <b>inp</b>. */ -config_line_t * -config_lines_dup(const config_line_t *inp) -{ - return config_lines_dup_and_filter(inp, NULL); -} - -/** Return a newly allocated deep copy of the lines in <b>inp</b>, - * but only the ones that match <b>key</b>. */ -static config_line_t * -config_lines_dup_and_filter(const config_line_t *inp, - const char *key) -{ - config_line_t *result = NULL; - config_line_t **next_out = &result; - while (inp) { - if (key && strcasecmpstart(inp->key, key)) { - inp = inp->next; - continue; - } - *next_out = tor_malloc_zero(sizeof(config_line_t)); - (*next_out)->key = tor_strdup(inp->key); - (*next_out)->value = tor_strdup(inp->value); - inp = inp->next; - next_out = &((*next_out)->next); - } - (*next_out) = NULL; - return result; -} - /** Return newly allocated line or lines corresponding to <b>key</b> in the * configuration <b>options</b>. If <b>escape_val</b> is true and a * value needs to be quoted before it's put in a config file, quote and @@ -1028,36 +882,6 @@ config_free(const config_format_t *fmt, void *options) tor_free(options); } -/** Return true iff a and b contain identical keys and values in identical - * order. */ -int -config_lines_eq(config_line_t *a, config_line_t *b) -{ - while (a && b) { - if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) - return 0; - a = a->next; - b = b->next; - } - if (a || b) - return 0; - return 1; -} - -/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */ -int -config_count_key(const config_line_t *a, const char *key) -{ - int n = 0; - while (a) { - if (!strcasecmp(a->key, key)) { - ++n; - } - a = a->next; - } - return n; -} - /** Return true iff the option <b>name</b> has the same value in <b>o1</b> * and <b>o2</b>. Must not be called for LINELIST_S or OBSOLETE options. */ @@ -1366,8 +1190,6 @@ config_parse_msec_interval(const char *s, int *ok) { uint64_t r; r = config_parse_units(s, time_msec_units, ok); - if (!ok) - return -1; if (r > INT_MAX) { log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); *ok = 0; @@ -1385,8 +1207,6 @@ config_parse_interval(const char *s, int *ok) { uint64_t r; r = config_parse_units(s, time_units, ok); - if (!ok) - return -1; if (r > INT_MAX) { log_warn(LD_CONFIG, "Interval '%s' is too long", s); *ok = 0; diff --git a/src/or/confparse.h b/src/or/confparse.h index 8d915d266b..9c4205d07c 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONFPARSE_H @@ -103,14 +103,7 @@ typedef struct config_format_t { #define CAL_WARN_DEPRECATIONS (1u<<2) void *config_new(const config_format_t *fmt); -void config_line_append(config_line_t **lst, - const char *key, const char *val); -config_line_t *config_lines_dup(const config_line_t *inp); -const config_line_t *config_line_find(const config_line_t *lines, - const char *key); void config_free(const config_format_t *fmt, void *options); -int config_lines_eq(config_line_t *a, config_line_t *b); -int config_count_key(const config_line_t *a, const char *key); config_line_t *config_get_assigned_option(const config_format_t *fmt, const void *options, const char *key, int escape_val); @@ -131,9 +124,6 @@ const char *config_find_deprecation(const config_format_t *fmt, const char *key); const config_var_t *config_find_option(const config_format_t *fmt, const char *key); - -int config_get_lines(const char *string, config_line_t **result, int extended); -void config_free_lines(config_line_t *front); const char *config_expand_abbrev(const config_format_t *fmt, const char *option, int command_line, int warn_obsolete); diff --git a/src/or/connection.c b/src/or/connection.c index 188276026f..4e890497e9 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -628,13 +628,13 @@ connection_free_(connection_t *conn) dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - tor_zlib_free(dir_conn->zlib_state); - if (dir_conn->fingerprint_stack) { - SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp)); - smartlist_free(dir_conn->fingerprint_stack); + tor_compress_free(dir_conn->compress_state); + if (dir_conn->spool) { + SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled, + spooled_resource_free(spooled)); + smartlist_free(dir_conn->spool); } - cached_dir_decref(dir_conn->cached_dir); rend_data_free(dir_conn->rend_data); if (dir_conn->guard_state) { /* Cancel before freeing, if it's still there. */ @@ -4038,10 +4038,6 @@ connection_flush(connection_t *conn) * its contents compressed or decompressed as they're written. If zlib is * negative, this is the last data to be compressed, and the connection's zlib * state should be flushed. - * - * If it's a local control connection and a 64k chunk is ready, try to flush - * it all, so we don't end up with many megabytes of controller info queued at - * once. */ MOCK_IMPL(void, connection_write_to_buf_impl_,(const char *string, size_t len, @@ -4060,9 +4056,9 @@ connection_write_to_buf_impl_,(const char *string, size_t len, if (zlib) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); int done = zlib < 0; - CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf, - dir_conn->zlib_state, - string, len, done)); + CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf, + dir_conn->compress_state, + string, len, done)); } else { CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf)); } diff --git a/src/or/connection.h b/src/or/connection.h index d25e002fa4..36e45aef38 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -141,17 +141,17 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); -/* DOCDOC connection_write_to_buf_zlib */ -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); +/* DOCDOC connection_write_to_buf_compress */ +static void connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done); static inline void connection_write_to_buf(const char *string, size_t len, connection_t *conn) { connection_write_to_buf_impl_(string, len, conn, 0); } static inline void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) +connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done) { connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1); } diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index d9d9e73643..8480a35458 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,7 +29,7 @@ * <li>DNS lookup streams, created on the exit side in response to * a RELAY_RESOLVE cell from a client. * <li>Tunneled directory streams, created on the directory cache side - * in response to a RELAY_BEGINDIR cell. These streams attach directly + * in response to a RELAY_BEGIN_DIR cell. These streams attach directly * to a dir_connection_t object without ever using TCP. * </ul> * @@ -261,6 +261,7 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) } /* Fall through if the connection is on a circuit without optimistic * data support. */ + /* Falls through. */ case EXIT_CONN_STATE_CONNECTING: case AP_CONN_STATE_RENDDESC_WAIT: case AP_CONN_STATE_CIRCUIT_WAIT: @@ -1762,7 +1763,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, conn->entry_cfg.ipv6_traffic = 0; /* Still handling CONNECT. Now, check for exit enclaves. (Which we - * don't do on BEGINDIR, or when there is a chosen exit.) + * don't do on BEGIN_DIR, or when there is a chosen exit.) * * TODO: Should we remove this? Exit enclaves are nutty and don't * work very well @@ -1987,8 +1988,8 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req) socklen_t orig_dst_len = sizeof(orig_dst); tor_addr_t addr; -#ifdef TRANS_TRPOXY - if (options->TransProxyType_parsed == TPT_TPROXY) { +#ifdef TRANS_TPROXY + if (get_options()->TransProxyType_parsed == TPT_TPROXY) { if (getsockname(ENTRY_TO_CONN(conn)->s, (struct sockaddr*)&orig_dst, &orig_dst_len) < 0) { int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); @@ -2527,7 +2528,7 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn) /* Sensitive directory connections must have an anonymous path length. * Otherwise, directory connections are typically one-hop. * This matches the earlier check for directory connection path anonymity - * in directory_initiate_command_rend(). */ + * in directory_initiate_request(). */ if (purpose_needs_anonymity(linked_dir_conn_base->purpose, TO_DIR_CONN(linked_dir_conn_base)->router_purpose, TO_DIR_CONN(linked_dir_conn_base)->requested_resource)) { @@ -3001,7 +3002,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, return; } -/** Read a RELAY_BEGIN or RELAY_BEGINDIR cell from <b>cell</b>, decode it, and +/** Read a RELAY_BEGIN or RELAY_BEGIN_DIR cell from <b>cell</b>, decode it, and * place the result in <b>bcell</b>. On success return 0; on failure return * <0 and set *<b>end_reason_out</b> to the end reason we should send back to * the client. @@ -3140,15 +3141,13 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) port = bcell.port; if (or_circ && or_circ->p_chan) { - if (!options->AllowSingleHopExits && - (or_circ->is_first_hop || - (!connection_or_digest_is_known_relay( + if ((or_circ->is_first_hop || + (!connection_or_digest_is_known_relay( or_circ->p_chan->identity_digest) && should_refuse_unknown_exits(options)))) { - /* Don't let clients use us as a single-hop proxy, unless the user - * has explicitly allowed that in the config. It attracts attackers - * and users who'd be better off with, well, single-hop proxies. - */ + /* Don't let clients use us as a single-hop proxy. It attracts + * attackers and users who'd be better off with, well, single-hop + * proxies. */ log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Attempt by %s to open a stream %s. Closing.", safe_str(channel_get_canonical_remote_descr(or_circ->p_chan)), diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index 61b5752aed..e4780b3c7d 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 6eb62a99de..ab0f411cc5 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -55,6 +55,7 @@ #include "ext_orport.h" #include "scheduler.h" #include "torcert.h" +#include "channelpadding.h" static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_launch_v3_or_handshake(or_connection_t *conn); @@ -814,24 +815,6 @@ connection_or_update_token_buckets(smartlist_t *conns, }); } -/** How long do we wait before killing non-canonical OR connections with no - * circuits? In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15 - * minutes before cancelling these connections, which caused fast relays to - * accrue many many idle connections. Hopefully 3-4.5 minutes is low enough - * that it kills most idle connections, without being so low that we cause - * clients to bounce on and off. - * - * For canonical connections, the limit is higher, at 15-22.5 minutes. - * - * For each OR connection, we randomly add up to 50% extra to its idle_timeout - * field, to avoid exposing when exactly the last circuit closed. Since we're - * storing idle_timeout in a uint16_t, don't let these values get higher than - * 12 hours or so without revising connection_or_set_canonical and/or expanding - * idle_timeout. - */ -#define IDLE_OR_CONN_TIMEOUT_NONCANONICAL 180 -#define IDLE_OR_CONN_TIMEOUT_CANONICAL 900 - /* Mark <b>or_conn</b> as canonical if <b>is_canonical</b> is set, and * non-canonical otherwise. Adjust idle_timeout accordingly. */ @@ -839,9 +822,6 @@ void connection_or_set_canonical(or_connection_t *or_conn, int is_canonical) { - const unsigned int timeout_base = is_canonical ? - IDLE_OR_CONN_TIMEOUT_CANONICAL : IDLE_OR_CONN_TIMEOUT_NONCANONICAL; - if (bool_eq(is_canonical, or_conn->is_canonical) && or_conn->idle_timeout != 0) { /* Don't recalculate an existing idle_timeout unless the canonical @@ -850,7 +830,14 @@ connection_or_set_canonical(or_connection_t *or_conn, } or_conn->is_canonical = !! is_canonical; /* force to a 1-bit boolean */ - or_conn->idle_timeout = timeout_base + crypto_rand_int(timeout_base / 2); + or_conn->idle_timeout = channelpadding_get_channel_idle_timeout( + TLS_CHAN_TO_BASE(or_conn->chan), is_canonical); + + log_info(LD_CIRC, + "Channel " U64_FORMAT " chose an idle timeout of %d.", + or_conn->chan ? + U64_PRINTF_ARG(TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier):0, + or_conn->idle_timeout); } /** If we don't necessarily know the router we're connecting to, but we @@ -1053,10 +1040,8 @@ connection_or_group_set_badness_(smartlist_t *group, int force) } if (!best || - channel_is_better(now, - TLS_CHAN_TO_BASE(or_conn->chan), - TLS_CHAN_TO_BASE(best->chan), - 0)) { + channel_is_better(TLS_CHAN_TO_BASE(or_conn->chan), + TLS_CHAN_TO_BASE(best->chan))) { best = or_conn; } } SMARTLIST_FOREACH_END(or_conn); @@ -1084,11 +1069,9 @@ connection_or_group_set_badness_(smartlist_t *group, int force) or_conn->base_.state != OR_CONN_STATE_OPEN) continue; if (or_conn != best && - channel_is_better(now, - TLS_CHAN_TO_BASE(best->chan), - TLS_CHAN_TO_BASE(or_conn->chan), 1)) { - /* This isn't the best conn, _and_ the best conn is better than it, - even when we're being forgiving. */ + channel_is_better(TLS_CHAN_TO_BASE(best->chan), + TLS_CHAN_TO_BASE(or_conn->chan))) { + /* This isn't the best conn, _and_ the best conn is better than it */ if (best->is_canonical) { log_info(LD_OR, "Marking OR conn to %s:%d as unsuitable for new circuits: " @@ -1987,12 +1970,23 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn) cell_pack(&networkcell, cell, conn->wide_circ_ids); + rep_hist_padding_count_write(PADDING_TYPE_TOTAL); + if (cell->command == CELL_PADDING) + rep_hist_padding_count_write(PADDING_TYPE_CELL); + connection_write_to_buf(networkcell.body, cell_network_size, TO_CONN(conn)); /* Touch the channel's active timestamp if there is one */ - if (conn->chan) + if (conn->chan) { channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); + if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) { + rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL); + if (cell->command == CELL_PADDING) + rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL); + } + } + if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_cell(conn, conn->handshake_state, cell, 0); } @@ -2098,7 +2092,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) } /** Array of recognized link protocol versions. */ -static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4 }; +static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4, 5 }; /** Number of versions in <b>or_protocol_versions</b>. */ static const int n_or_protocol_versions = (int)( sizeof(or_protocol_versions)/sizeof(uint16_t) ); diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 514a0fd008..fe85a3f5fd 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -109,6 +109,8 @@ void var_cell_free(var_cell_t *cell); /* DOCDOC */ #define MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS 4 +#define MIN_LINK_PROTO_FOR_CHANNEL_PADDING 5 +#define MAX_LINK_PROTO MIN_LINK_PROTO_FOR_CHANNEL_PADDING void connection_or_group_set_badness_(smartlist_t *group, int force); diff --git a/src/or/conscache.c b/src/or/conscache.c new file mode 100644 index 0000000000..5ffa129bbe --- /dev/null +++ b/src/or/conscache.c @@ -0,0 +1,541 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" + +#include "config.h" +#include "conscache.h" +#include "storagedir.h" + +#define CCE_MAGIC 0x17162253 + +/** + * A consensus_cache_entry_t is a reference-counted handle to an + * item in a consensus_cache_t. It can be mmapped into RAM, or not, + * depending whether it's currently in use. + */ +struct consensus_cache_entry_t { + uint32_t magic; /**< Must be set to CCE_MAGIC */ + HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t); + int32_t refcnt; /**< Reference count. */ + unsigned can_remove : 1; /**< If true, we want to delete this file. */ + /** If true, we intend to unmap this file as soon as we're done with it. */ + unsigned release_aggressively : 1; + + /** Filename for this object within the storage_dir_t */ + char *fname; + /** Labels associated with this object. Immutable once the object + * is created. */ + config_line_t *labels; + /** Pointer to the cache that includes this entry (if any). */ + consensus_cache_t *in_cache; + + /** Since what time has this object been mapped into RAM, but with the cache + * being the only having a reference to it? */ + time_t unused_since; + /** mmaped contents of the underlying file. May be NULL */ + tor_mmap_t *map; + /** Length of the body within <b>map</b>. */ + size_t bodylen; + /** Pointer to the body within <b>map</b>. */ + const uint8_t *body; +}; + +/** + * A consensus_cache_t holds a directory full of labeled items. + */ +struct consensus_cache_t { + /** Underling storage_dir_t to handle persistence */ + storage_dir_t *dir; + /** List of all the entries in the directory. */ + smartlist_t *entries; +}; + +static void consensus_cache_clear(consensus_cache_t *cache); +static void consensus_cache_rescan(consensus_cache_t *); +static void consensus_cache_entry_map(consensus_cache_t *, + consensus_cache_entry_t *); +static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent); + +/** + * Helper: Open a consensus cache in subdirectory <b>subdir</b> of the + * data directory, to hold up to <b>max_entries</b> of data. + */ +consensus_cache_t * +consensus_cache_open(const char *subdir, int max_entries) +{ + consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t)); + char *directory = get_datadir_fname(subdir); + cache->dir = storage_dir_new(directory, max_entries); + tor_free(directory); + if (!cache->dir) { + tor_free(cache); + return NULL; + } + + consensus_cache_rescan(cache); + return cache; +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that <b>cache</b> will need. + */ +int +consensus_cache_register_with_sandbox(consensus_cache_t *cache, + struct sandbox_cfg_elem **cfg) +{ + return storage_dir_register_with_sandbox(cache->dir, cfg); +} + +/** + * Helper: clear all entries from <b>cache</b> (but do not delete + * any that aren't marked for removal + */ +static void +consensus_cache_clear(consensus_cache_t *cache) +{ + consensus_cache_delete_pending(cache, 0); + + SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { + ent->in_cache = NULL; + consensus_cache_entry_decref(ent); + } SMARTLIST_FOREACH_END(ent); + smartlist_free(cache->entries); + cache->entries = NULL; +} + +/** + * Drop all storage held by <b>cache</b>. + */ +void +consensus_cache_free(consensus_cache_t *cache) +{ + if (! cache) + return; + + if (cache->entries) { + consensus_cache_clear(cache); + } + storage_dir_free(cache->dir); + tor_free(cache); +} + +/** + * Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>, + * labeling that data with <b>labels</b>. On failure, return NULL. On + * success, return a newly created consensus_cache_entry_t. + * + * The returned value will be owned by the cache, and you will have a + * reference to it. Call consensus_cache_entry_decref() when you are + * done with it. + * + * The provided <b>labels</b> MUST have distinct keys: if they don't, + * this API does not specify which values (if any) for the duplicate keys + * will be considered. + */ +consensus_cache_entry_t * +consensus_cache_add(consensus_cache_t *cache, + const config_line_t *labels, + const uint8_t *data, + size_t datalen) +{ + char *fname = NULL; + int r = storage_dir_save_labeled_to_file(cache->dir, + labels, data, datalen, &fname); + if (r < 0 || fname == NULL) { + return NULL; + } + consensus_cache_entry_t *ent = + tor_malloc_zero(sizeof(consensus_cache_entry_t)); + ent->magic = CCE_MAGIC; + ent->fname = fname; + ent->labels = config_lines_dup(labels); + ent->in_cache = cache; + ent->unused_since = TIME_MAX; + smartlist_add(cache->entries, ent); + /* Start the reference count at 2: the caller owns one copy, and the + * cache owns another. + */ + ent->refcnt = 2; + + return ent; +} + +/** + * Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>. + * Return NULL if no such entry exists. + * + * Does not adjust reference counts. + */ +consensus_cache_entry_t * +consensus_cache_find_first(consensus_cache_t *cache, + const char *key, + const char *value) +{ + smartlist_t *tmp = smartlist_new(); + consensus_cache_find_all(tmp, cache, key, value); + consensus_cache_entry_t *ent = NULL; + if (smartlist_len(tmp)) + ent = smartlist_get(tmp, 0); + smartlist_free(tmp); + return ent; +} + +/** + * Given a <b>cache</b>, add every entry to <b>out<b> for which + * <b>key</b>=<b>value</b>. If <b>key</b> is NULL, add every entry. + * + * Do not add any entry that has been marked for removal. + * + * Does not adjust reference counts. + */ +void +consensus_cache_find_all(smartlist_t *out, + consensus_cache_t *cache, + const char *key, + const char *value) +{ + SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { + if (ent->can_remove == 1) { + /* We want to delete this; pretend it isn't there. */ + continue; + } + if (! key) { + smartlist_add(out, ent); + continue; + } + const char *found_val = consensus_cache_entry_get_value(ent, key); + if (found_val && !strcmp(value, found_val)) { + smartlist_add(out, ent); + } + } SMARTLIST_FOREACH_END(ent); +} + +/** + * Given a list of consensus_cache_entry_t, remove all those entries + * that do not have <b>key</b>=<b>value</b> in their labels. + * + * Does not adjust reference counts. + */ +void +consensus_cache_filter_list(smartlist_t *lst, + const char *key, + const char *value) +{ + if (BUG(lst == NULL)) + return; // LCOV_EXCL_LINE + if (key == NULL) + return; + SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) { + const char *found_val = consensus_cache_entry_get_value(ent, key); + if (! found_val || strcmp(value, found_val)) { + SMARTLIST_DEL_CURRENT(lst, ent); + } + } SMARTLIST_FOREACH_END(ent); +} + +/** + * If <b>ent</b> has a label with the given <b>key</b>, return its + * value. Otherwise return NULL. + * + * The return value is only guaranteed to be valid for as long as you + * hold a reference to <b>ent</b>. + */ +const char * +consensus_cache_entry_get_value(const consensus_cache_entry_t *ent, + const char *key) +{ + const config_line_t *match = config_line_find(ent->labels, key); + if (match) + return match->value; + else + return NULL; +} + +/** + * Return a pointer to the labels in <b>ent</b>. + * + * This pointer is only guaranteed to be valid for as long as you + * hold a reference to <b>ent</b>. + */ +const config_line_t * +consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent) +{ + return ent->labels; +} + +/** + * Increase the reference count of <b>ent</b>. + */ +void +consensus_cache_entry_incref(consensus_cache_entry_t *ent) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return; // LCOV_EXCL_LINE + ++ent->refcnt; + ent->unused_since = TIME_MAX; +} + +/** + * Release a reference held to <b>ent</b>. + * + * If it was the last reference, ent will be freed. Therefore, you must not + * use <b>ent</b> after calling this function. + */ +void +consensus_cache_entry_decref(consensus_cache_entry_t *ent) +{ + if (! ent) + return; + if (BUG(ent->refcnt <= 0)) + return; // LCOV_EXCL_LINE + if (BUG(ent->magic != CCE_MAGIC)) + return; // LCOV_EXCL_LINE + + --ent->refcnt; + + if (ent->refcnt == 1 && ent->in_cache) { + /* Only the cache has a reference: we don't need to keep the file + * mapped */ + if (ent->map) { + if (ent->release_aggressively) { + consensus_cache_entry_unmap(ent); + } else { + ent->unused_since = approx_time(); + } + } + return; + } + + if (ent->refcnt > 0) + return; + + /* Refcount is zero; we can free it. */ + if (ent->map) { + consensus_cache_entry_unmap(ent); + } + tor_free(ent->fname); + config_free_lines(ent->labels); + consensus_cache_entry_handles_clear(ent); + memwipe(ent, 0, sizeof(consensus_cache_entry_t)); + tor_free(ent); +} + +/** + * Mark <b>ent</b> for deletion from the cache. Deletion will not occur + * until the cache is the only place that holds a reference to <b>ent</b>. + */ +void +consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent) +{ + ent->can_remove = 1; +} + +/** + * Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for + * any longer than we're actually using it. + */ +void +consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent) +{ + ent->release_aggressively = 1; +} + +/** + * Try to read the body of <b>ent</b> into memory if it isn't already + * loaded. On success, set *<b>body_out</b> to the body, *<b>sz_out</b> + * to its size, and return 0. On failure return -1. + * + * The resulting body pointer will only be valid for as long as you + * hold a reference to <b>ent</b>. + */ +int +consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, + const uint8_t **body_out, + size_t *sz_out) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + if (! ent->map) { + if (! ent->in_cache) + return -1; + + consensus_cache_entry_map((consensus_cache_t *)ent->in_cache, + (consensus_cache_entry_t *)ent); + if (! ent->map) { + return -1; + } + } + + *body_out = ent->body; + *sz_out = ent->bodylen; + return 0; +} + +/** + * Unmap every mmap'd element of <b>cache</b> that has been unused + * since <b>cutoff</b>. + */ +void +consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff) +{ + SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { + tor_assert_nonfatal(ent->in_cache == cache); + if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) { + /* Somebody is using this entry right now */ + continue; + } + if (ent->unused_since > cutoff) { + /* Has been unused only for a little while */ + continue; + } + if (ent->map == NULL) { + /* Not actually mapped. */ + continue; + } + consensus_cache_entry_unmap(ent); + } SMARTLIST_FOREACH_END(ent); +} + +/** + * Return the number of currently unused filenames available in this cache. + */ +int +consensus_cache_get_n_filenames_available(consensus_cache_t *cache) +{ + tor_assert(cache); + int max = storage_dir_get_max_files(cache->dir); + int used = smartlist_len(storage_dir_list(cache->dir)); + tor_assert_nonfatal(max >= used); + return max - used; +} + +/** + * Delete every element of <b>cache</b> has been marked with + * consensus_cache_entry_mark_for_removal. If <b>force</b> is false, + * retain those entries which are not in use except by the cache. + */ +void +consensus_cache_delete_pending(consensus_cache_t *cache, int force) +{ + SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { + tor_assert_nonfatal(ent->in_cache == cache); + if (! force) { + if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) { + /* Somebody is using this entry right now */ + continue; + } + } + if (ent->can_remove == 0) { + /* Don't want to delete this. */ + continue; + } + if (BUG(ent->refcnt <= 0)) { + continue; // LCOV_EXCL_LINE + } + + SMARTLIST_DEL_CURRENT(cache->entries, ent); + ent->in_cache = NULL; + char *fname = tor_strdup(ent->fname); /* save a copy */ + consensus_cache_entry_decref(ent); + storage_dir_remove_file(cache->dir, fname); + tor_free(fname); + } SMARTLIST_FOREACH_END(ent); +} + +/** + * Internal helper: rescan <b>cache</b> and rebuild its list of entries. + */ +static void +consensus_cache_rescan(consensus_cache_t *cache) +{ + if (cache->entries) { + consensus_cache_clear(cache); + } + + cache->entries = smartlist_new(); + const smartlist_t *fnames = storage_dir_list(cache->dir); + SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) { + tor_mmap_t *map = NULL; + config_line_t *labels = NULL; + const uint8_t *body; + size_t bodylen; + map = storage_dir_map_labeled(cache->dir, fname, + &labels, &body, &bodylen); + if (! map) { + /* Can't load this; continue */ + log_warn(LD_FS, "Unable to map file %s from consensus cache: %s", + escaped(fname), strerror(errno)); + continue; + } + consensus_cache_entry_t *ent = + tor_malloc_zero(sizeof(consensus_cache_entry_t)); + ent->magic = CCE_MAGIC; + ent->fname = tor_strdup(fname); + ent->labels = labels; + ent->refcnt = 1; + ent->in_cache = cache; + ent->unused_since = TIME_MAX; + smartlist_add(cache->entries, ent); + tor_munmap_file(map); /* don't actually need to keep this around */ + } SMARTLIST_FOREACH_END(fname); +} + +/** + * Make sure that <b>ent</b> is mapped into RAM. + */ +static void +consensus_cache_entry_map(consensus_cache_t *cache, + consensus_cache_entry_t *ent) +{ + if (ent->map) + return; + + ent->map = storage_dir_map_labeled(cache->dir, ent->fname, + NULL, &ent->body, &ent->bodylen); + ent->unused_since = TIME_MAX; +} + +/** + * Unmap <b>ent</b> from RAM. + * + * Do not call this if something other than the cache is holding a reference + * to <b>ent</b> + */ +static void +consensus_cache_entry_unmap(consensus_cache_entry_t *ent) +{ + ent->unused_since = TIME_MAX; + if (!ent->map) + return; + + tor_munmap_file(ent->map); + ent->map = NULL; + ent->body = NULL; + ent->bodylen = 0; + ent->unused_since = TIME_MAX; +} + +HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, ) + +#ifdef TOR_UNIT_TESTS +/** + * Testing only: Return true iff <b>ent</b> is mapped into memory. + * + * (In normal operation, this information is not exposed.) + */ +int +consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent) +{ + if (ent->map) { + tor_assert(ent->body); + return 1; + } else { + tor_assert(!ent->body); + return 0; + } +} +#endif + diff --git a/src/or/conscache.h b/src/or/conscache.h new file mode 100644 index 0000000000..aef54201f0 --- /dev/null +++ b/src/or/conscache.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONSCACHE_H +#define TOR_CONSCACHE_H + +#include "handles.h" + +typedef struct consensus_cache_entry_t consensus_cache_entry_t; +typedef struct consensus_cache_t consensus_cache_t; + +HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, ) + +consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries); +void consensus_cache_free(consensus_cache_t *cache); +struct sandbox_cfg_elem; +int consensus_cache_register_with_sandbox(consensus_cache_t *cache, + struct sandbox_cfg_elem **cfg); +void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff); +void consensus_cache_delete_pending(consensus_cache_t *cache, + int force); +int consensus_cache_get_n_filenames_available(consensus_cache_t *cache); +consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache, + const config_line_t *labels, + const uint8_t *data, + size_t datalen); + +consensus_cache_entry_t *consensus_cache_find_first( + consensus_cache_t *cache, + const char *key, + const char *value); + +void consensus_cache_find_all(smartlist_t *out, + consensus_cache_t *cache, + const char *key, + const char *value); +void consensus_cache_filter_list(smartlist_t *lst, + const char *key, + const char *value); + +const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent, + const char *key); +const config_line_t *consensus_cache_entry_get_labels( + const consensus_cache_entry_t *ent); + +void consensus_cache_entry_incref(consensus_cache_entry_t *ent); +void consensus_cache_entry_decref(consensus_cache_entry_t *ent); + +void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent); +void consensus_cache_entry_mark_for_aggressive_release( + consensus_cache_entry_t *ent); +int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, + const uint8_t **body_out, + size_t *sz_out); + +#ifdef TOR_UNIT_TESTS +int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent); +#endif + +#endif + diff --git a/src/or/consdiff.c b/src/or/consdiff.c new file mode 100644 index 0000000000..1baa11897c --- /dev/null +++ b/src/or/consdiff.c @@ -0,0 +1,1412 @@ +/* Copyright (c) 2014, Daniel Martà + * Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file consdiff.c + * \brief Consensus diff implementation, including both the generation and the + * application of diffs in a minimal ed format. + * + * The consensus diff application is done in consdiff_apply_diff, which relies + * on apply_ed_diff for the main ed diff part and on some digest helper + * functions to check the digest hashes found in the consensus diff header. + * + * The consensus diff generation is more complex. consdiff_gen_diff generates + * it, relying on gen_ed_diff to generate the ed diff and some digest helper + * functions to generate the digest hashes. + * + * gen_ed_diff is the tricky bit. In it simplest form, it will take quadratic + * time and linear space to generate an ed diff given two smartlists. As shown + * in its comment section, calling calc_changes on the entire two consensuses + * will calculate what is to be added and what is to be deleted in the diff. + * Its comment section briefly explains how it works. + * + * In our case specific to consensuses, we take advantage of the fact that + * consensuses list routers sorted by their identities. We use that + * information to avoid running calc_changes on the whole smartlists. + * gen_ed_diff will navigate through the two consensuses identity by identity + * and will send small couples of slices to calc_changes, keeping the running + * time near-linear. This is explained in more detail in the gen_ed_diff + * comments. + * + * The allocation strategy tries to save time and memory by avoiding needless + * copies. Instead of actually splitting the inputs into separate strings, we + * allocate cdline_t objects, each of which represents a line in the original + * object or in the output. We use memarea_t allocators to manage the + * temporary memory we use when generating or applying diffs. + **/ + +#define CONSDIFF_PRIVATE + +#include "or.h" +#include "consdiff.h" +#include "memarea.h" +#include "routerparse.h" + +static const char* ns_diff_version = "network-status-diff-version 1"; +static const char* hash_token = "hash"; + +static char *consensus_join_lines(const smartlist_t *inp); + +/** Return true iff a and b have the same contents. */ +STATIC int +lines_eq(const cdline_t *a, const cdline_t *b) +{ + return a->len == b->len && fast_memeq(a->s, b->s, a->len); +} + +/** Return true iff a has the same contents as the nul-terminated string b. */ +STATIC int +line_str_eq(const cdline_t *a, const char *b) +{ + const size_t len = strlen(b); + tor_assert(len <= UINT32_MAX); + cdline_t bline = { b, (uint32_t)len }; + return lines_eq(a, &bline); +} + +/** Return true iff a begins with the same contents as the nul-terminated + * string b. */ +static int +line_starts_with_str(const cdline_t *a, const char *b) +{ + const size_t len = strlen(b); + tor_assert(len <= UINT32_MAX); + return a->len >= len && fast_memeq(a->s, b, len); +} + +/** Return a new cdline_t holding as its contents the nul-terminated + * string s. Use the provided memory area for storage. */ +static cdline_t * +cdline_linecpy(memarea_t *area, const char *s) +{ + size_t len = strlen(s); + const char *ss = memarea_memdup(area, s, len); + cdline_t *line = memarea_alloc(area, sizeof(cdline_t)); + line->s = ss; + line->len = (uint32_t)len; + return line; +} + +/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated + * string s. Use the provided memory area for storage. */ +STATIC void +smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) +{ + smartlist_add(lst, cdline_linecpy(area, s)); +} + +/** Compute the digest of <b>cons</b>, and store the result in + * <b>digest_out</b>. Return 0 on success, -1 on failure. */ +/* This is a separate, mockable function so that we can override it when + * fuzzing. */ +MOCK_IMPL(STATIC int, +consensus_compute_digest,(const char *cons, + consensus_digest_t *digest_out)) +{ + int r = crypto_digest256((char*)digest_out->sha3_256, + cons, strlen(cons), DIGEST_SHA3_256); + return r; +} + +/** Compute the digest-as-signed of <b>cons</b>, and store the result in + * <b>digest_out</b>. Return 0 on success, -1 on failure. */ +/* This is a separate, mockable function so that we can override it when + * fuzzing. */ +MOCK_IMPL(STATIC int, +consensus_compute_digest_as_signed,(const char *cons, + consensus_digest_t *digest_out)) +{ + return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256, + cons); +} + +/** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */ +/* This is a separate, mockable function so that we can override it when + * fuzzing. */ +MOCK_IMPL(STATIC int, +consensus_digest_eq,(const uint8_t *d1, + const uint8_t *d2)) +{ + return fast_memeq(d1, d2, DIGEST256_LEN); +} + +/** Create (allocate) a new slice from a smartlist. Assumes that the start + * and the end indexes are within the bounds of the initial smartlist. The end + * element is not part of the resulting slice. If end is -1, the slice is to + * reach the end of the smartlist. + */ +STATIC smartlist_slice_t * +smartlist_slice(const smartlist_t *list, int start, int end) +{ + int list_len = smartlist_len(list); + tor_assert(start >= 0); + tor_assert(start <= list_len); + if (end == -1) { + end = list_len; + } + tor_assert(start <= end); + + smartlist_slice_t *slice = tor_malloc(sizeof(smartlist_slice_t)); + slice->list = list; + slice->offset = start; + slice->len = end - start; + return slice; +} + +/** Helper: Compute the longest common subsequence lengths for the two slices. + * Used as part of the diff generation to find the column at which to split + * slice2 while still having the optimal solution. + * If direction is -1, the navigation is reversed. Otherwise it must be 1. + * The length of the resulting integer array is that of the second slice plus + * one. + */ +STATIC int * +lcs_lengths(const smartlist_slice_t *slice1, const smartlist_slice_t *slice2, + int direction) +{ + size_t a_size = sizeof(int) * (slice2->len+1); + + /* Resulting lcs lengths. */ + int *result = tor_malloc_zero(a_size); + /* Copy of the lcs lengths from the last iteration. */ + int *prev = tor_malloc(a_size); + + tor_assert(direction == 1 || direction == -1); + + int si = slice1->offset; + if (direction == -1) { + si += (slice1->len-1); + } + + for (int i = 0; i < slice1->len; ++i, si+=direction) { + + const cdline_t *line1 = smartlist_get(slice1->list, si); + /* Store the last results. */ + memcpy(prev, result, a_size); + + int sj = slice2->offset; + if (direction == -1) { + sj += (slice2->len-1); + } + + for (int j = 0; j < slice2->len; ++j, sj+=direction) { + + const cdline_t *line2 = smartlist_get(slice2->list, sj); + if (lines_eq(line1, line2)) { + /* If the lines are equal, the lcs is one line longer. */ + result[j + 1] = prev[j] + 1; + } else { + /* If not, see what lcs parent path is longer. */ + result[j + 1] = MAX(result[j], prev[j + 1]); + } + } + } + tor_free(prev); + return result; +} + +/** Helper: Trim any number of lines that are equally at the start or the end + * of both slices. + */ +STATIC void +trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2) +{ + while (slice1->len>0 && slice2->len>0) { + const cdline_t *line1 = smartlist_get(slice1->list, slice1->offset); + const cdline_t *line2 = smartlist_get(slice2->list, slice2->offset); + if (!lines_eq(line1, line2)) { + break; + } + slice1->offset++; slice1->len--; + slice2->offset++; slice2->len--; + } + + int i1 = (slice1->offset+slice1->len)-1; + int i2 = (slice2->offset+slice2->len)-1; + + while (slice1->len>0 && slice2->len>0) { + const cdline_t *line1 = smartlist_get(slice1->list, i1); + const cdline_t *line2 = smartlist_get(slice2->list, i2); + if (!lines_eq(line1, line2)) { + break; + } + i1--; + slice1->len--; + i2--; + slice2->len--; + } +} + +/** Like smartlist_string_pos, but uses a cdline_t, and is restricted to the + * bounds of the slice. + */ +STATIC int +smartlist_slice_string_pos(const smartlist_slice_t *slice, + const cdline_t *string) +{ + int end = slice->offset + slice->len; + for (int i = slice->offset; i < end; ++i) { + const cdline_t *el = smartlist_get(slice->list, i); + if (lines_eq(el, string)) { + return i; + } + } + return -1; +} + +/** Helper: Set all the appropriate changed booleans to true. The first slice + * must be of length 0 or 1. All the lines of slice1 and slice2 which are not + * present in the other slice will be set to changed in their bool array. + * The two changed bool arrays are passed in the same order as the slices. + */ +STATIC void +set_changed(bitarray_t *changed1, bitarray_t *changed2, + const smartlist_slice_t *slice1, const smartlist_slice_t *slice2) +{ + int toskip = -1; + tor_assert(slice1->len == 0 || slice1->len == 1); + + if (slice1->len == 1) { + const cdline_t *line_common = smartlist_get(slice1->list, slice1->offset); + toskip = smartlist_slice_string_pos(slice2, line_common); + if (toskip == -1) { + bitarray_set(changed1, slice1->offset); + } + } + int end = slice2->offset + slice2->len; + for (int i = slice2->offset; i < end; ++i) { + if (i != toskip) { + bitarray_set(changed2, i); + } + } +} + +/* + * Helper: Given that slice1 has been split by half into top and bot, we want + * to fetch the column at which to split slice2 so that we are still on track + * to the optimal diff solution, i.e. the shortest one. We use lcs_lengths + * since the shortest diff is just another way to say the longest common + * subsequence. + */ +static int +optimal_column_to_split(const smartlist_slice_t *top, + const smartlist_slice_t *bot, + const smartlist_slice_t *slice2) +{ + int *lens_top = lcs_lengths(top, slice2, 1); + int *lens_bot = lcs_lengths(bot, slice2, -1); + int column=0, max_sum=-1; + + for (int i = 0; i < slice2->len+1; ++i) { + int sum = lens_top[i] + lens_bot[slice2->len-i]; + if (sum > max_sum) { + column = i; + max_sum = sum; + } + } + tor_free(lens_top); + tor_free(lens_bot); + + return column; +} + +/** + * Helper: Figure out what elements are new or gone on the second smartlist + * relative to the first smartlist, and store the booleans in the bitarrays. + * True on the first bitarray means the element is gone, true on the second + * bitarray means it's new. + * + * In its base case, either of the smartlists is of length <= 1 and we can + * quickly see what elements are new or are gone. In the other case, we will + * split one smartlist by half and we'll use optimal_column_to_split to find + * the optimal column at which to split the second smartlist so that we are + * finding the smallest diff possible. + */ +STATIC void +calc_changes(smartlist_slice_t *slice1, + smartlist_slice_t *slice2, + bitarray_t *changed1, bitarray_t *changed2) +{ + trim_slices(slice1, slice2); + + if (slice1->len <= 1) { + set_changed(changed1, changed2, slice1, slice2); + + } else if (slice2->len <= 1) { + set_changed(changed2, changed1, slice2, slice1); + + /* Keep on splitting the slices in two. */ + } else { + smartlist_slice_t *top, *bot, *left, *right; + + /* Split the first slice in half. */ + int mid = slice1->len/2; + top = smartlist_slice(slice1->list, slice1->offset, slice1->offset+mid); + bot = smartlist_slice(slice1->list, slice1->offset+mid, + slice1->offset+slice1->len); + + /* Split the second slice by the optimal column. */ + int mid2 = optimal_column_to_split(top, bot, slice2); + left = smartlist_slice(slice2->list, slice2->offset, slice2->offset+mid2); + right = smartlist_slice(slice2->list, slice2->offset+mid2, + slice2->offset+slice2->len); + + calc_changes(top, left, changed1, changed2); + calc_changes(bot, right, changed1, changed2); + tor_free(top); + tor_free(bot); + tor_free(left); + tor_free(right); + } +} + +/* This table is from crypto.c. The SP and PAD defines are different. */ +#define NOT_VALID_BASE64 255 +#define X NOT_VALID_BASE64 +#define SP NOT_VALID_BASE64 +#define PAD NOT_VALID_BASE64 +static const uint8_t base64_compare_table[256] = { + X, X, X, X, X, X, X, X, X, SP, SP, SP, X, SP, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + SP, X, X, X, X, X, X, X, X, X, X, 62, X, X, X, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, X, X, X, PAD, X, X, + X, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, X, X, X, X, X, + X, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, +}; + +/** Helper: Get the identity hash from a router line, assuming that the line + * at least appears to be a router line and thus starts with "r ". + * + * If an identity hash is found, store it (without decoding it) in + * <b>hash_out</b>, and return 0. On failure, return -1. + */ +STATIC int +get_id_hash(const cdline_t *line, cdline_t *hash_out) +{ + if (line->len < 2) + return -1; + + /* Skip the router name. */ + const char *hash = memchr(line->s + 2, ' ', line->len - 2); + if (!hash) { + return -1; + } + + hash++; + const char *hash_end = hash; + /* Stop when the first non-base64 character is found. Use unsigned chars to + * avoid negative indexes causing crashes. + */ + while (base64_compare_table[*((unsigned char*)hash_end)] + != NOT_VALID_BASE64 && + hash_end < line->s + line->len) { + hash_end++; + } + + /* Empty hash. */ + if (hash_end == hash) { + return -1; + } + + hash_out->s = hash; + /* Always true because lines are limited to this length */ + tor_assert(hash_end >= hash); + tor_assert((size_t)(hash_end - hash) <= UINT32_MAX); + hash_out->len = (uint32_t)(hash_end - hash); + + return 0; +} + +/** Helper: Check that a line is a valid router entry. We must at least be + * able to fetch a proper identity hash from it for it to be valid. + */ +STATIC int +is_valid_router_entry(const cdline_t *line) +{ + if (line->len < 2 || fast_memneq(line->s, "r ", 2)) + return 0; + cdline_t tmp; + return (get_id_hash(line, &tmp) == 0); +} + +/** Helper: Find the next router line starting at the current position. + * Assumes that cur is lower than the length of the smartlist, i.e. it is a + * line within the bounds of the consensus. The only exception is when we + * don't want to skip the first line, in which case cur will be -1. + */ +STATIC int +next_router(const smartlist_t *cons, int cur) +{ + int len = smartlist_len(cons); + tor_assert(cur >= -1 && cur < len); + + if (++cur >= len) { + return len; + } + + const cdline_t *line = smartlist_get(cons, cur); + while (!is_valid_router_entry(line)) { + if (++cur >= len) { + return len; + } + line = smartlist_get(cons, cur); + } + return cur; +} + +/** Helper: compare two base64-encoded identity hashes, which may be of + * different lengths. Comparison ends when the first non-base64 char is found. + */ +STATIC int +base64cmp(const cdline_t *hash1, const cdline_t *hash2) +{ + /* NULL is always lower, useful for last_hash which starts at NULL. */ + if (!hash1->s && !hash2->s) { + return 0; + } + if (!hash1->s) { + return -1; + } + if (!hash2->s) { + return 1; + } + + /* Don't index with a char; char may be signed. */ + const unsigned char *a = (unsigned char*)hash1->s; + const unsigned char *b = (unsigned char*)hash2->s; + const unsigned char *a_end = a + hash1->len; + const unsigned char *b_end = b + hash2->len; + while (1) { + uint8_t av = base64_compare_table[*a]; + uint8_t bv = base64_compare_table[*b]; + if (av == NOT_VALID_BASE64) { + if (bv == NOT_VALID_BASE64) { + /* Both ended with exactly the same characters. */ + return 0; + } else { + /* hash2 goes on longer than hash1 and thus hash1 is lower. */ + return -1; + } + } else if (bv == NOT_VALID_BASE64) { + /* hash1 goes on longer than hash2 and thus hash1 is greater. */ + return 1; + } else if (av < bv) { + /* The first difference shows that hash1 is lower. */ + return -1; + } else if (av > bv) { + /* The first difference shows that hash1 is greater. */ + return 1; + } else { + a++; + b++; + if (a == a_end) { + if (b == b_end) { + return 0; + } else { + return -1; + } + } else if (b == b_end) { + return 1; + } + } + } +} + +/** Structure used to remember the previous and current identity hash of + * the "r " lines in a consensus, to enforce well-ordering. */ +typedef struct router_id_iterator_t { + cdline_t last_hash; + cdline_t hash; +} router_id_iterator_t; + +/** + * Initializer for a router_id_iterator_t. + */ +#define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } } + +/** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance + * the index to the next router line ("r ...") in the consensus, or to + * an index one after the end of the list if there is no such line. + * + * Use <b>iter</b> to record the hash of the found router line, if any, + * and to enforce ordering on the hashes. If the hashes are mis-ordered, + * return -1. Else, return 0. + **/ +static int +find_next_router_line(const smartlist_t *cons, + const char *consname, + int *idxp, + router_id_iterator_t *iter) +{ + *idxp = next_router(cons, *idxp); + if (*idxp < smartlist_len(cons)) { + memcpy(&iter->last_hash, &iter->hash, sizeof(cdline_t)); + if (get_id_hash(smartlist_get(cons, *idxp), &iter->hash) < 0 || + base64cmp(&iter->hash, &iter->last_hash) <= 0) { + log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because " + "the %s consensus doesn't have its router entries sorted " + "properly.", consname); + return -1; + } + } + return 0; +} + +/** Line-prefix indicating the beginning of the signatures section that we + * intend to delete. */ +#define START_OF_SIGNATURES_SECTION "directory-signature " + +/** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t) + * to remove the signatures from it. If the footer is removed, return a + * cdline_t containing a delete command to delete the footer, allocated in + * <b>area</>. If no footer is removed, return NULL. + * + * We remove the signatures here because they are not themselves signed, and + * as such there might be different encodings for them. + */ +static cdline_t * +preprocess_consensus(memarea_t *area, + smartlist_t *cons) +{ + int idx; + int dirsig_idx = -1; + for (idx = 0; idx < smartlist_len(cons); ++idx) { + cdline_t *line = smartlist_get(cons, idx); + if (line_starts_with_str(line, START_OF_SIGNATURES_SECTION)) { + dirsig_idx = idx; + break; + } + } + if (dirsig_idx >= 0) { + char buf[64]; + while (smartlist_len(cons) > dirsig_idx) + smartlist_del(cons, dirsig_idx); + tor_snprintf(buf, sizeof(buf), "%d,$d", dirsig_idx+1); + return cdline_linecpy(area, buf); + } else { + return NULL; + } +} + +/** Generate an ed diff as a smartlist from two consensuses, also given as + * smartlists. Will return NULL if the diff could not be generated, which can + * happen if any lines the script had to add matched "." or if the routers + * were not properly ordered. + * + * All cdline_t objects in the resulting object are either references to lines + * in one of the inputs, or are newly allocated lines in the provided memarea. + * + * This implementation is consensus-specific. To generate an ed diff for any + * given input in quadratic time, you can replace all the code until the + * navigation in reverse order with the following: + * + * int len1 = smartlist_len(cons1); + * int len2 = smartlist_len(cons2); + * bitarray_t *changed1 = bitarray_init_zero(len1); + * bitarray_t *changed2 = bitarray_init_zero(len2); + * cons1_sl = smartlist_slice(cons1, 0, -1); + * cons2_sl = smartlist_slice(cons2, 0, -1); + * calc_changes(cons1_sl, cons2_sl, changed1, changed2); + */ +STATIC smartlist_t * +gen_ed_diff(const smartlist_t *cons1_orig, const smartlist_t *cons2, + memarea_t *area) +{ + smartlist_t *cons1 = smartlist_new(); + smartlist_add_all(cons1, cons1_orig); + cdline_t *remove_trailer = preprocess_consensus(area, cons1); + + int len1 = smartlist_len(cons1); + int len2 = smartlist_len(cons2); + smartlist_t *result = smartlist_new(); + + if (remove_trailer) { + /* There's a delete-the-trailer line at the end, so add it here. */ + smartlist_add(result, remove_trailer); + } + + /* Initialize the changed bitarrays to zero, so that calc_changes only needs + * to set the ones that matter and leave the rest untouched. + */ + bitarray_t *changed1 = bitarray_init_zero(len1); + bitarray_t *changed2 = bitarray_init_zero(len2); + int i1=-1, i2=-1; + int start1=0, start2=0; + + /* To check that hashes are ordered properly */ + router_id_iterator_t iter1 = ROUTER_ID_ITERATOR_INIT; + router_id_iterator_t iter2 = ROUTER_ID_ITERATOR_INIT; + + /* i1 and i2 are initialized at the first line of each consensus. They never + * reach past len1 and len2 respectively, since next_router doesn't let that + * happen. i1 and i2 are advanced by at least one line at each iteration as + * long as they have not yet reached len1 and len2, so the loop is + * guaranteed to end, and each pair of (i1,i2) will be inspected at most + * once. + */ + while (i1 < len1 || i2 < len2) { + + /* Advance each of the two navigation positions by one router entry if not + * yet at the end. + */ + if (i1 < len1) { + if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) { + goto error_cleanup; + } + } + + if (i2 < len2) { + if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) { + goto error_cleanup; + } + } + + /* If we have reached the end of both consensuses, there is no need to + * compare hashes anymore, since this is the last iteration. + */ + if (i1 < len1 || i2 < len2) { + + /* Keep on advancing the lower (by identity hash sorting) position until + * we have two matching positions. The only other possible outcome is + * that a lower position reaches the end of the consensus before it can + * reach a hash that is no longer the lower one. Since there will always + * be a lower hash for as long as the loop runs, one of the two indexes + * will always be incremented, thus assuring that the loop must end + * after a finite number of iterations. If that cannot be because said + * consensus has already reached the end, both are extended to their + * respecting ends since we are done. + */ + int cmp = base64cmp(&iter1.hash, &iter2.hash); + while (cmp != 0) { + if (i1 < len1 && cmp < 0) { + if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) { + goto error_cleanup; + } + if (i1 == len1) { + /* We finished the first consensus, so grab all the remaining + * lines of the second consensus and finish up. + */ + i2 = len2; + break; + } + } else if (i2 < len2 && cmp > 0) { + if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) { + goto error_cleanup; + } + if (i2 == len2) { + /* We finished the second consensus, so grab all the remaining + * lines of the first consensus and finish up. + */ + i1 = len1; + break; + } + } else { + i1 = len1; + i2 = len2; + break; + } + cmp = base64cmp(&iter1.hash, &iter2.hash); + } + } + + /* Make slices out of these chunks (up to the common router entry) and + * calculate the changes for them. + * Error if any of the two slices are longer than 10K lines. That should + * never happen with any pair of real consensuses. Feeding more than 10K + * lines to calc_changes would be very slow anyway. + */ +#define MAX_LINE_COUNT (10000) + if (i1-start1 > MAX_LINE_COUNT || i2-start2 > MAX_LINE_COUNT) { + log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because " + "we found too few common router ids."); + goto error_cleanup; + } + + smartlist_slice_t *cons1_sl = smartlist_slice(cons1, start1, i1); + smartlist_slice_t *cons2_sl = smartlist_slice(cons2, start2, i2); + calc_changes(cons1_sl, cons2_sl, changed1, changed2); + tor_free(cons1_sl); + tor_free(cons2_sl); + start1 = i1, start2 = i2; + } + + /* Navigate the changes in reverse order and generate one ed command for + * each chunk of changes. + */ + i1=len1-1, i2=len2-1; + char buf[128]; + while (i1 >= 0 || i2 >= 0) { + + int start1x, start2x, end1, end2, added, deleted; + + /* We are at a point were no changed bools are true, so just keep going. */ + if (!(i1 >= 0 && bitarray_is_set(changed1, i1)) && + !(i2 >= 0 && bitarray_is_set(changed2, i2))) { + if (i1 >= 0) { + i1--; + } + if (i2 >= 0) { + i2--; + } + continue; + } + + end1 = i1, end2 = i2; + + /* Grab all contiguous changed lines */ + while (i1 >= 0 && bitarray_is_set(changed1, i1)) { + i1--; + } + while (i2 >= 0 && bitarray_is_set(changed2, i2)) { + i2--; + } + + start1x = i1+1, start2x = i2+1; + added = end2-i2, deleted = end1-i1; + + if (added == 0) { + if (deleted == 1) { + tor_snprintf(buf, sizeof(buf), "%id", start1x+1); + smartlist_add_linecpy(result, area, buf); + } else { + tor_snprintf(buf, sizeof(buf), "%i,%id", start1x+1, start1x+deleted); + smartlist_add_linecpy(result, area, buf); + } + } else { + int i; + if (deleted == 0) { + tor_snprintf(buf, sizeof(buf), "%ia", start1x); + smartlist_add_linecpy(result, area, buf); + } else if (deleted == 1) { + tor_snprintf(buf, sizeof(buf), "%ic", start1x+1); + smartlist_add_linecpy(result, area, buf); + } else { + tor_snprintf(buf, sizeof(buf), "%i,%ic", start1x+1, start1x+deleted); + smartlist_add_linecpy(result, area, buf); + } + + for (i = start2x; i <= end2; ++i) { + cdline_t *line = smartlist_get(cons2, i); + if (line_str_eq(line, ".")) { + log_warn(LD_CONSDIFF, "Cannot generate consensus diff because " + "one of the lines to be added is \".\"."); + goto error_cleanup; + } + smartlist_add(result, line); + } + smartlist_add_linecpy(result, area, "."); + } + } + + smartlist_free(cons1); + bitarray_free(changed1); + bitarray_free(changed2); + + return result; + + error_cleanup: + + smartlist_free(cons1); + bitarray_free(changed1); + bitarray_free(changed2); + + smartlist_free(result); + + return NULL; +} + +/* Helper: Read a base-10 number between 0 and INT32_MAX from <b>s</b> and + * store it in <b>num_out</b>. Advance <b>s</b> to the characer immediately + * after the number. Return 0 on success, -1 on failure. */ +static int +get_linenum(const char **s, int *num_out) +{ + int ok; + char *next; + if (!TOR_ISDIGIT(**s)) { + return -1; + } + *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next); + if (ok && next) { + *s = next; + return 0; + } else { + return -1; + } +} + +/** Apply the ed diff, starting at <b>diff_starting_line</b>, to the consensus + * and return a new consensus, also as a line-based smartlist. Will return + * NULL if the ed diff is not properly formatted. + * + * All cdline_t objects in the resulting object are references to lines + * in one of the inputs; nothing is copied. + */ +STATIC smartlist_t * +apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, + int diff_starting_line) +{ + int diff_len = smartlist_len(diff); + int j = smartlist_len(cons1); + smartlist_t *cons2 = smartlist_new(); + + for (int i=diff_starting_line; i<diff_len; ++i) { + const cdline_t *diff_cdline = smartlist_get(diff, i); + char diff_line[128]; + + if (diff_cdline->len > sizeof(diff_line) - 1) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command was far too long"); + goto error_cleanup; + } + /* Copy the line to make it nul-terminated. */ + memcpy(diff_line, diff_cdline->s, diff_cdline->len); + diff_line[diff_cdline->len] = 0; + const char *ptr = diff_line; + int start = 0, end = 0; + int had_range = 0; + int end_was_eof = 0; + if (get_linenum(&ptr, &start) < 0) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command was missing a line number."); + goto error_cleanup; + } + if (*ptr == ',') { + /* Two-item range */ + had_range = 1; + ++ptr; + if (*ptr == '$') { + end_was_eof = 1; + end = smartlist_len(cons1); + ++ptr; + } else if (get_linenum(&ptr, &end) < 0) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command was missing a range end line number."); + goto error_cleanup; + } + /* Incoherent range. */ + if (end <= start) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an invalid range was found in an ed command."); + goto error_cleanup; + } + } else { + /* We'll take <n1> as <n1>,<n1> for simplicity. */ + end = start; + } + + if (end > j) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "its commands are not properly sorted in reverse order."); + goto error_cleanup; + } + + if (*ptr == '\0') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "a line with no ed command was found"); + goto error_cleanup; + } + + if (*(ptr+1) != '\0') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command longer than one char was found."); + goto error_cleanup; + } + + char action = *ptr; + + switch (action) { + case 'a': + case 'c': + case 'd': + break; + default: + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an unrecognised ed command was found."); + goto error_cleanup; + } + + /** $ is not allowed with non-d actions. */ + if (end_was_eof && action != 'd') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it wanted to use $ with a command other than delete"); + goto error_cleanup; + } + + /* 'a' commands are not allowed to have ranges. */ + if (had_range && action == 'a') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it wanted to add lines after a range."); + goto error_cleanup; + } + + /* Add unchanged lines. */ + for (; j && j > end; --j) { + cdline_t *cons_line = smartlist_get(cons1, j-1); + smartlist_add(cons2, cons_line); + } + + /* Ignore removed lines. */ + if (action == 'c' || action == 'd') { + while (--j >= start) { + /* Skip line */ + } + } + + /* Add new lines in reverse order, since it will all be reversed at the + * end. + */ + if (action == 'a' || action == 'c') { + int added_end = i; + + i++; /* Skip the line with the range and command. */ + while (i < diff_len) { + if (line_str_eq(smartlist_get(diff, i), ".")) { + break; + } + if (++i == diff_len) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it has lines to be inserted that don't end with a \".\"."); + goto error_cleanup; + } + } + + int added_i = i-1; + + /* It would make no sense to add zero new lines. */ + if (added_i == added_end) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it has an ed command that tries to insert zero lines."); + goto error_cleanup; + } + + while (added_i > added_end) { + cdline_t *added_line = smartlist_get(diff, added_i--); + smartlist_add(cons2, added_line); + } + } + } + + /* Add remaining unchanged lines. */ + for (; j > 0; --j) { + cdline_t *cons_line = smartlist_get(cons1, j-1); + smartlist_add(cons2, cons_line); + } + + /* Reverse the whole thing since we did it from the end. */ + smartlist_reverse(cons2); + return cons2; + + error_cleanup: + + smartlist_free(cons2); + + return NULL; +} + +/** Generate a consensus diff as a smartlist from two given consensuses, also + * as smartlists. Will return NULL if the consensus diff could not be + * generated. Neither of the two consensuses are modified in any way, so it's + * up to the caller to free their resources. + */ +smartlist_t * +consdiff_gen_diff(const smartlist_t *cons1, + const smartlist_t *cons2, + const consensus_digest_t *digests1, + const consensus_digest_t *digests2, + memarea_t *area) +{ + smartlist_t *ed_diff = gen_ed_diff(cons1, cons2, area); + /* ed diff could not be generated - reason already logged by gen_ed_diff. */ + if (!ed_diff) { + goto error_cleanup; + } + + /* See that the script actually produces what we want. */ + smartlist_t *ed_cons2 = apply_ed_diff(cons1, ed_diff, 0); + if (!ed_cons2) { + /* LCOV_EXCL_START -- impossible if diff generation is correct */ + log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because " + "the generated ed diff could not be tested to successfully generate " + "the target consensus."); + goto error_cleanup; + /* LCOV_EXCL_STOP */ + } + + int cons2_eq = 1; + if (smartlist_len(cons2) == smartlist_len(ed_cons2)) { + SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) { + const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx); + if (! lines_eq(line1, line2) ) { + cons2_eq = 0; + break; + } + } SMARTLIST_FOREACH_END(line1); + } else { + cons2_eq = 0; + } + smartlist_free(ed_cons2); + if (!cons2_eq) { + /* LCOV_EXCL_START -- impossible if diff generation is correct. */ + log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because " + "the generated ed diff did not generate the target consensus " + "successfully when tested."); + goto error_cleanup; + /* LCOV_EXCL_STOP */ + } + + char cons1_hash_hex[HEX_DIGEST256_LEN+1]; + char cons2_hash_hex[HEX_DIGEST256_LEN+1]; + base16_encode(cons1_hash_hex, HEX_DIGEST256_LEN+1, + (const char*)digests1->sha3_256, DIGEST256_LEN); + base16_encode(cons2_hash_hex, HEX_DIGEST256_LEN+1, + (const char*)digests2->sha3_256, DIGEST256_LEN); + + /* Create the resulting consensus diff. */ + char buf[160]; + smartlist_t *result = smartlist_new(); + tor_snprintf(buf, sizeof(buf), "%s", ns_diff_version); + smartlist_add_linecpy(result, area, buf); + tor_snprintf(buf, sizeof(buf), "%s %s %s", hash_token, + cons1_hash_hex, cons2_hash_hex); + smartlist_add_linecpy(result, area, buf); + smartlist_add_all(result, ed_diff); + smartlist_free(ed_diff); + return result; + + error_cleanup: + + if (ed_diff) { + /* LCOV_EXCL_START -- ed_diff is NULL except in unreachable cases above */ + smartlist_free(ed_diff); + /* LCOV_EXCL_STOP */ + } + + return NULL; +} + +/** Fetch the digest of the base consensus in the consensus diff, encoded in + * base16 as found in the diff itself. digest1_out and digest2_out must be of + * length DIGEST256_LEN or larger if not NULL. + */ +int +consdiff_get_digests(const smartlist_t *diff, + char *digest1_out, + char *digest2_out) +{ + smartlist_t *hash_words = NULL; + const cdline_t *format; + char cons1_hash[DIGEST256_LEN], cons2_hash[DIGEST256_LEN]; + char *cons1_hash_hex, *cons2_hash_hex; + if (smartlist_len(diff) < 2) { + log_info(LD_CONSDIFF, "The provided consensus diff is too short."); + goto error_cleanup; + } + + /* Check that it's the format and version we know. */ + format = smartlist_get(diff, 0); + if (!line_str_eq(format, ns_diff_version)) { + log_warn(LD_CONSDIFF, "The provided consensus diff format is not known."); + goto error_cleanup; + } + + /* Grab the base16 digests. */ + hash_words = smartlist_new(); + { + const cdline_t *line2 = smartlist_get(diff, 1); + char *h = tor_memdup_nulterm(line2->s, line2->len); + smartlist_split_string(hash_words, h, " ", 0, 0); + tor_free(h); + } + + /* There have to be three words, the first of which must be hash_token. */ + if (smartlist_len(hash_words) != 3 || + strcmp(smartlist_get(hash_words, 0), hash_token)) { + log_info(LD_CONSDIFF, "The provided consensus diff does not include " + "the necessary digests."); + goto error_cleanup; + } + + /* Expected hashes as found in the consensus diff header. They must be of + * length HEX_DIGEST256_LEN, normally 64 hexadecimal characters. + * If any of the decodings fail, error to make sure that the hashes are + * proper base16-encoded digests. + */ + cons1_hash_hex = smartlist_get(hash_words, 1); + cons2_hash_hex = smartlist_get(hash_words, 2); + if (strlen(cons1_hash_hex) != HEX_DIGEST256_LEN || + strlen(cons2_hash_hex) != HEX_DIGEST256_LEN) { + log_info(LD_CONSDIFF, "The provided consensus diff includes " + "base16-encoded digests of incorrect size."); + goto error_cleanup; + } + + if (base16_decode(cons1_hash, DIGEST256_LEN, + cons1_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN || + base16_decode(cons2_hash, DIGEST256_LEN, + cons2_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN) { + log_info(LD_CONSDIFF, "The provided consensus diff includes " + "malformed digests."); + goto error_cleanup; + } + + if (digest1_out) { + memcpy(digest1_out, cons1_hash, DIGEST256_LEN); + } + if (digest2_out) { + memcpy(digest2_out, cons2_hash, DIGEST256_LEN); + } + + SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp)); + smartlist_free(hash_words); + return 0; + + error_cleanup: + + if (hash_words) { + SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp)); + smartlist_free(hash_words); + } + return 1; +} + +/** Apply the consensus diff to the given consensus and return a new + * consensus, also as a line-based smartlist. Will return NULL if the diff + * could not be applied. Neither the consensus nor the diff are modified in + * any way, so it's up to the caller to free their resources. + */ +char * +consdiff_apply_diff(const smartlist_t *cons1, + const smartlist_t *diff, + const consensus_digest_t *digests1) +{ + smartlist_t *cons2 = NULL; + char *cons2_str = NULL; + char e_cons1_hash[DIGEST256_LEN]; + char e_cons2_hash[DIGEST256_LEN]; + + if (consdiff_get_digests(diff, e_cons1_hash, e_cons2_hash) != 0) { + goto error_cleanup; + } + + /* See that the consensus that was given to us matches its hash. */ + if (!consensus_digest_eq(digests1->sha3_256, + (const uint8_t*)e_cons1_hash)) { + char hex_digest1[HEX_DIGEST256_LEN+1]; + char e_hex_digest1[HEX_DIGEST256_LEN+1]; + log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because " + "the base consensus doesn't match the digest as found in " + "the consensus diff header."); + base16_encode(hex_digest1, HEX_DIGEST256_LEN+1, + (const char *)digests1->sha3_256, DIGEST256_LEN); + base16_encode(e_hex_digest1, HEX_DIGEST256_LEN+1, + e_cons1_hash, DIGEST256_LEN); + log_warn(LD_CONSDIFF, "Expected: %s; found: %s", + hex_digest1, e_hex_digest1); + goto error_cleanup; + } + + /* Grab the ed diff and calculate the resulting consensus. */ + /* Skip the first two lines. */ + cons2 = apply_ed_diff(cons1, diff, 2); + + /* ed diff could not be applied - reason already logged by apply_ed_diff. */ + if (!cons2) { + goto error_cleanup; + } + + cons2_str = consensus_join_lines(cons2); + + consensus_digest_t cons2_digests; + if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) { + /* LCOV_EXCL_START -- digest can't fail */ + log_warn(LD_CONSDIFF, "Could not compute digests of the consensus " + "resulting from applying a consensus diff."); + goto error_cleanup; + /* LCOV_EXCL_STOP */ + } + + /* See that the resulting consensus matches its hash. */ + if (!consensus_digest_eq(cons2_digests.sha3_256, + (const uint8_t*)e_cons2_hash)) { + log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because " + "the resulting consensus doesn't match the digest as found in " + "the consensus diff header."); + char hex_digest2[HEX_DIGEST256_LEN+1]; + char e_hex_digest2[HEX_DIGEST256_LEN+1]; + base16_encode(hex_digest2, HEX_DIGEST256_LEN+1, + (const char *)cons2_digests.sha3_256, DIGEST256_LEN); + base16_encode(e_hex_digest2, HEX_DIGEST256_LEN+1, + e_cons2_hash, DIGEST256_LEN); + log_warn(LD_CONSDIFF, "Expected: %s; found: %s", + hex_digest2, e_hex_digest2); + goto error_cleanup; + } + + goto done; + + error_cleanup: + tor_free(cons2_str); /* Sets it to NULL */ + + done: + if (cons2) { + smartlist_free(cons2); + } + + return cons2_str; +} + +/** Any consensus line longer than this means that the input is invalid. */ +#define CONSENSUS_LINE_MAX_LEN (1<<20) + +/** + * Helper: For every NL-terminated line in <b>s</b>, add a cdline referring to + * that line (without trailing newline) to <b>out</b>. Return -1 if there are + * any non-NL terminated lines; 0 otherwise. + * + * Unlike tor_split_lines, this function avoids ambiguity on its + * handling of a final line that isn't NL-terminated. + * + * All cdline_t objects are allocated in the provided memarea. Strings + * are not copied: if <b>s</b> changes or becomes invalid, then all + * generated cdlines will become invalid. + */ +STATIC int +consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area) +{ + while (*s) { + const char *eol = strchr(s, '\n'); + if (!eol) { + /* File doesn't end with newline. */ + return -1; + } + if (eol - s > CONSENSUS_LINE_MAX_LEN) { + /* Line is far too long. */ + return -1; + } + cdline_t *line = memarea_alloc(area, sizeof(cdline_t)); + line->s = s; + line->len = (uint32_t)(eol - s); + smartlist_add(out, line); + s = eol+1; + } + return 0; +} + +/** Given a list of cdline_t, return a newly allocated string containing + * all of the lines, terminated with NL, concatenated. + * + * Unlike smartlist_join_strings(), avoids lossy operations on empty + * lists. */ +static char * +consensus_join_lines(const smartlist_t *inp) +{ + size_t n = 0; + SMARTLIST_FOREACH(inp, const cdline_t *, cdline, n += cdline->len + 1); + n += 1; + char *result = tor_malloc(n); + char *out = result; + SMARTLIST_FOREACH_BEGIN(inp, const cdline_t *, cdline) { + memcpy(out, cdline->s, cdline->len); + out += cdline->len; + *out++ = '\n'; + } SMARTLIST_FOREACH_END(cdline); + *out++ = '\0'; + tor_assert(out == result+n); + return result; +} + +/** Given two consensus documents, try to compute a diff between them. On + * success, retun a newly allocated string containing that diff. On failure, + * return NULL. */ +char * +consensus_diff_generate(const char *cons1, + const char *cons2) +{ + consensus_digest_t d1, d2; + smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL; + int r1, r2; + char *result = NULL; + + r1 = consensus_compute_digest_as_signed(cons1, &d1); + r2 = consensus_compute_digest(cons2, &d2); + if (BUG(r1 < 0 || r2 < 0)) + return NULL; // LCOV_EXCL_LINE + + memarea_t *area = memarea_new(); + lines1 = smartlist_new(); + lines2 = smartlist_new(); + if (consensus_split_lines(lines1, cons1, area) < 0) + goto done; + if (consensus_split_lines(lines2, cons2, area) < 0) + goto done; + + result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area); + + done: + if (result_lines) { + result = consensus_join_lines(result_lines); + smartlist_free(result_lines); + } + + memarea_drop_all(area); + smartlist_free(lines1); + smartlist_free(lines2); + + return result; +} + +/** Given a consensus document and a diff, try to apply the diff to the + * consensus. On success return a newly allocated string containing the new + * consensus. On failure, return NULL. */ +char * +consensus_diff_apply(const char *consensus, + const char *diff) +{ + consensus_digest_t d1; + smartlist_t *lines1 = NULL, *lines2 = NULL; + int r1; + char *result = NULL; + memarea_t *area = memarea_new(); + + r1 = consensus_compute_digest_as_signed(consensus, &d1); + if (BUG(r1 < 0)) + return NULL; // LCOV_EXCL_LINE + + lines1 = smartlist_new(); + lines2 = smartlist_new(); + if (consensus_split_lines(lines1, consensus, area) < 0) + goto done; + if (consensus_split_lines(lines2, diff, area) < 0) + goto done; + + result = consdiff_apply_diff(lines1, lines2, &d1); + + done: + smartlist_free(lines1); + smartlist_free(lines2); + memarea_drop_all(area); + + return result; +} + +/** Return true iff, based on its header, <b>document</b> is likely + * to be a consensus diff. */ +int +looks_like_a_consensus_diff(const char *document, size_t len) +{ + return (len >= strlen(ns_diff_version) && + fast_memeq(document, ns_diff_version, strlen(ns_diff_version))); +} + diff --git a/src/or/consdiff.h b/src/or/consdiff.h new file mode 100644 index 0000000000..d05df74b75 --- /dev/null +++ b/src/or/consdiff.h @@ -0,0 +1,98 @@ +/* Copyright (c) 2014, Daniel Martà + * Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONSDIFF_H +#define TOR_CONSDIFF_H + +#include "or.h" + +char *consensus_diff_generate(const char *cons1, + const char *cons2); +char *consensus_diff_apply(const char *consensus, + const char *diff); + +int looks_like_a_consensus_diff(const char *document, size_t len); + +#ifdef CONSDIFF_PRIVATE +struct memarea_t; + +/** Line type used for constructing consensus diffs. Each of these lines + * refers to a chunk of memory allocated elsewhere, and is not necessarily + * NUL-terminated: this helps us avoid copies and save memory. */ +typedef struct cdline_t { + const char *s; + uint32_t len; +} cdline_t; + +typedef struct consensus_digest_t { + uint8_t sha3_256[DIGEST256_LEN]; +} consensus_digest_t; + +STATIC smartlist_t *consdiff_gen_diff(const smartlist_t *cons1, + const smartlist_t *cons2, + const consensus_digest_t *digests1, + const consensus_digest_t *digests2, + struct memarea_t *area); +STATIC char *consdiff_apply_diff(const smartlist_t *cons1, + const smartlist_t *diff, + const consensus_digest_t *digests1); +STATIC int consdiff_get_digests(const smartlist_t *diff, + char *digest1_out, + char *digest2_out); + +/** Data structure to define a slice of a smarltist. */ +typedef struct smartlist_slice_t { + /** + * Smartlist that this slice is made from. + * References the whole original smartlist that the slice was made out of. + * */ + const smartlist_t *list; + /** Starting position of the slice in the smartlist. */ + int offset; + /** Length of the slice, i.e. the number of elements it holds. */ + int len; +} smartlist_slice_t; +STATIC smartlist_t *gen_ed_diff(const smartlist_t *cons1, + const smartlist_t *cons2, + struct memarea_t *area); +STATIC smartlist_t *apply_ed_diff(const smartlist_t *cons1, + const smartlist_t *diff, + int start_line); +STATIC void calc_changes(smartlist_slice_t *slice1, smartlist_slice_t *slice2, + bitarray_t *changed1, bitarray_t *changed2); +STATIC smartlist_slice_t *smartlist_slice(const smartlist_t *list, + int start, int end); +STATIC int next_router(const smartlist_t *cons, int cur); +STATIC int *lcs_lengths(const smartlist_slice_t *slice1, + const smartlist_slice_t *slice2, + int direction); +STATIC void trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2); +STATIC int base64cmp(const cdline_t *hash1, const cdline_t *hash2); +STATIC int get_id_hash(const cdline_t *line, cdline_t *hash_out); +STATIC int is_valid_router_entry(const cdline_t *line); +STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice, + const cdline_t *string); +STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2, + const smartlist_slice_t *slice1, + const smartlist_slice_t *slice2); +STATIC int consensus_split_lines(smartlist_t *out, const char *s, + struct memarea_t *area); +STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area, + const char *s); +STATIC int lines_eq(const cdline_t *a, const cdline_t *b); +STATIC int line_str_eq(const cdline_t *a, const char *b); + +MOCK_DECL(STATIC int, + consensus_compute_digest,(const char *cons, + consensus_digest_t *digest_out)); +MOCK_DECL(STATIC int, + consensus_compute_digest_as_signed,(const char *cons, + consensus_digest_t *digest_out)); +MOCK_DECL(STATIC int, + consensus_digest_eq,(const uint8_t *d1, + const uint8_t *d2)); +#endif + +#endif + diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c new file mode 100644 index 0000000000..2af104733b --- /dev/null +++ b/src/or/consdiffmgr.c @@ -0,0 +1,1842 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file consdiffmsr.c + * + * \brief consensus diff manager functions + * + * This module is run by directory authorities and caches in order + * to remember a number of past consensus documents, and to generate + * and serve the diffs from those documents to the latest consensus. + */ + +#define CONSDIFFMGR_PRIVATE + +#include "or.h" +#include "conscache.h" +#include "consdiff.h" +#include "consdiffmgr.h" +#include "cpuworker.h" +#include "networkstatus.h" +#include "routerparse.h" +#include "workqueue.h" + +/** + * Labels to apply to items in the conscache object. + * + * @{ + */ +/* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */ +#define LABEL_DOCTYPE "document-type" +/* The valid-after time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_VALID_AFTER "consensus-valid-after" +/* The fresh-until time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_FRESH_UNTIL "consensus-fresh-until" +/* The valid-until time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_VALID_UNTIL "consensus-valid-until" +/* Comma-separated list of hex-encoded identity digests for the voting + * authorities. */ +#define LABEL_SIGNATORIES "consensus-signatories" +/* A hex encoded SHA3 digest of the object, as compressed (if any) */ +#define LABEL_SHA3_DIGEST "sha3-digest" +/* A hex encoded SHA3 digest of the object before compression. */ +#define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed" +/* A hex encoded SHA3 digest-as-signed of a consensus */ +#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed" +/* The flavor of the consensus or consensuses diff */ +#define LABEL_FLAVOR "consensus-flavor" +/* Diff only: the SHA3 digest-as-signed of the source consensus. */ +#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest" +/* Diff only: the SHA3 digest-in-full of the target consensus. */ +#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest" +/* Diff only: the valid-after date of the source consensus. */ +#define LABEL_FROM_VALID_AFTER "from-valid-after" +/* What kind of compression was used? */ +#define LABEL_COMPRESSION_TYPE "compression" +/** @} */ + +#define DOCTYPE_CONSENSUS "consensus" +#define DOCTYPE_CONSENSUS_DIFF "consensus-diff" + +/** + * Underlying directory that stores consensuses and consensus diffs. Don't + * use this directly: use cdm_cache_get() instead. + */ +static consensus_cache_t *cons_diff_cache = NULL; +/** + * If true, we have learned at least one new consensus since the + * consensus cache was last up-to-date. + */ +static int cdm_cache_dirty = 0; +/** + * If true, we have scanned the cache to update our hashtable of diffs. + */ +static int cdm_cache_loaded = 0; + +/** + * Possible status values for cdm_diff_t.cdm_diff_status + **/ +typedef enum cdm_diff_status_t { + CDM_DIFF_PRESENT=1, + CDM_DIFF_IN_PROGRESS=2, + CDM_DIFF_ERROR=3, +} cdm_diff_status_t; + +/** Which methods do we use for precompressing diffs? */ +static const compress_method_t compress_diffs_with[] = { + NO_METHOD, + GZIP_METHOD, +#ifdef HAVE_LZMA + LZMA_METHOD, +#endif +#ifdef HAVE_ZSTD + ZSTD_METHOD, +#endif +}; + +/** How many different methods will we try to use for diff compression? */ +STATIC unsigned +n_diff_compression_methods(void) +{ + return ARRAY_LENGTH(compress_diffs_with); +} + +/** Which methods do we use for precompressing consensuses? */ +static const compress_method_t compress_consensus_with[] = { + ZLIB_METHOD, +#ifdef HAVE_LZMA + LZMA_METHOD, +#endif +#ifdef HAVE_ZSTD + ZSTD_METHOD, +#endif +}; + +/** How many different methods will we try to use for diff compression? */ +STATIC unsigned +n_consensus_compression_methods(void) +{ + return ARRAY_LENGTH(compress_consensus_with); +} + +/** For which compression method do we retain old consensuses? There's no + * need to keep all of them, since we won't be serving them. We'll + * go with ZLIB_METHOD because it's pretty fast and everyone has it. + */ +#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD + +/** Handles pointing to the latest consensus entries as compressed and + * stored. */ +static consensus_cache_entry_handle_t * + latest_consensus[N_CONSENSUS_FLAVORS] + [ARRAY_LENGTH(compress_consensus_with)]; + +/** Hashtable node used to remember the current status of the diff + * from a given sha3 digest to the current consensus. */ +typedef struct cdm_diff_t { + HT_ENTRY(cdm_diff_t) node; + + /** Consensus flavor for this diff (part of ht key) */ + consensus_flavor_t flavor; + /** SHA3-256 digest of the consensus that this diff is _from_. (part of the + * ht key) */ + uint8_t from_sha3[DIGEST256_LEN]; + /** Method by which the diff is compressed. (part of the ht key */ + compress_method_t compress_method; + + /** One of the CDM_DIFF_* values, depending on whether this diff + * is available, in progress, or impossible to compute. */ + cdm_diff_status_t cdm_diff_status; + /** SHA3-256 digest of the consensus that this diff is _to. */ + uint8_t target_sha3[DIGEST256_LEN]; + + /** Handle to the cache entry for this diff, if any. We use a handle here + * to avoid thinking too hard about cache entry lifetime issues. */ + consensus_cache_entry_handle_t *entry; +} cdm_diff_t; + +/** Hashtable mapping flavor and source consensus digest to status. */ +static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER(); + +/** + * Configuration for this module + */ +static consdiff_cfg_t consdiff_cfg = { + // XXXX I'd like to make this number bigger, but it interferes with the + // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096) + // XXXX rules. + /* .cache_max_num = */ 128 +}; + +static int consdiffmgr_ensure_space_for_files(int n); +static int consensus_queue_compression_work(const char *consensus, + const networkstatus_t *as_parsed); +static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, + consensus_cache_entry_t *diff_to); +static void consdiffmgr_set_cache_flags(void); + +/* ===== + * Hashtable setup + * ===== */ + +/** Helper: hash the key of a cdm_diff_t. */ +static unsigned +cdm_diff_hash(const cdm_diff_t *diff) +{ + uint8_t tmp[DIGEST256_LEN + 2]; + memcpy(tmp, diff->from_sha3, DIGEST256_LEN); + tmp[DIGEST256_LEN] = (uint8_t) diff->flavor; + tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method; + return (unsigned) siphash24g(tmp, sizeof(tmp)); +} +/** Helper: compare two cdm_diff_t objects for key equality */ +static int +cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2) +{ + return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) && + diff1->flavor == diff2->flavor && + diff1->compress_method == diff2->compress_method; +} + +HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq) +HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq, + 0.6, tor_reallocarray, tor_free_) + +/** Release all storage held in <b>diff</b>. */ +static void +cdm_diff_free(cdm_diff_t *diff) +{ + if (!diff) + return; + consensus_cache_entry_handle_free(diff->entry); + tor_free(diff); +} + +/** Create and return a new cdm_diff_t with the given values. Does not + * add it to the hashtable. */ +static cdm_diff_t * +cdm_diff_new(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *target_sha3, + compress_method_t method) +{ + cdm_diff_t *ent; + ent = tor_malloc_zero(sizeof(cdm_diff_t)); + ent->flavor = flav; + memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN); + memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN); + ent->compress_method = method; + return ent; +} + +/** + * Examine the diff hashtable to see whether we know anything about computing + * a diff of type <b>flav</b> between consensuses with the two provided + * SHA3-256 digests. If a computation is in progress, or if the computation + * has already been tried and failed, return 1. Otherwise, note the + * computation as "in progress" so that we don't reattempt it later, and + * return 0. + */ +static int +cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *target_sha3) +{ + struct cdm_diff_t search, *ent; + unsigned u; + int result = 0; + for (u = 0; u < n_diff_compression_methods(); ++u) { + compress_method_t method = compress_diffs_with[u]; + memset(&search, 0, sizeof(cdm_diff_t)); + search.flavor = flav; + search.compress_method = method; + memcpy(search.from_sha3, from_sha3, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + if (ent) { + tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT); + result = 1; + continue; + } + ent = cdm_diff_new(flav, from_sha3, target_sha3, method); + ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS; + HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent); + } + return result; +} + +/** + * Update the status of the diff of type <b>flav</b> between consensuses with + * the two provided SHA3-256 digests, so that its status becomes + * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b> + * is NULL, then the old handle (if any) is freed, and replaced with NULL. + */ +static void +cdm_diff_ht_set_status(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *to_sha3, + compress_method_t method, + int status, + consensus_cache_entry_handle_t *handle) +{ + struct cdm_diff_t search, *ent; + memset(&search, 0, sizeof(cdm_diff_t)); + search.flavor = flav; + search.compress_method = method, + memcpy(search.from_sha3, from_sha3, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + if (!ent) { + ent = cdm_diff_new(flav, from_sha3, to_sha3, method); + ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS; + HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent); + } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) { + // This can happen under certain really pathological conditions + // if we decide we don't care about a diff before it is actually + // done computing. + return; + } + + tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS); + + ent->cdm_diff_status = status; + consensus_cache_entry_handle_free(ent->entry); + ent->entry = handle; +} + +/** + * Helper: Remove from the hash table every present (actually computed) diff + * of type <b>flav</b> whose target digest does not match + * <b>unless_target_sha3_matches</b>. + * + * This function is used for the hash table to throw away references to diffs + * that do not lead to the most given consensus of a given flavor. + */ +static void +cdm_diff_ht_purge(consensus_flavor_t flav, + const uint8_t *unless_target_sha3_matches) +{ + cdm_diff_t **diff, **next; + for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) { + cdm_diff_t *this = *diff; + + if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT && + flav == (*diff)->flavor) { + + if (consensus_cache_entry_handle_get((*diff)->entry) == NULL) { + /* the underlying entry has gone away; drop this. */ + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + continue; + } + + if (unless_target_sha3_matches && + fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3, + DIGEST256_LEN)) { + /* target hash doesn't match; drop this. */ + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + continue; + } + } + next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff); + } +} + +/** + * Helper: initialize <b>cons_diff_cache</b>. + */ +static void +cdm_cache_init(void) +{ + unsigned n_entries = consdiff_cfg.cache_max_num * 2; + + tor_assert(cons_diff_cache == NULL); + cons_diff_cache = consensus_cache_open("diff-cache", n_entries); + if (cons_diff_cache == NULL) { + // LCOV_EXCL_START + log_err(LD_FS, "Error: Couldn't open storage for consensus diffs."); + tor_assert_unreached(); + // LCOV_EXCL_STOP + } else { + consdiffmgr_set_cache_flags(); + } + cdm_cache_dirty = 1; + cdm_cache_loaded = 0; +} + +/** + * Helper: return the consensus_cache_t * that backs this manager, + * initializing it if needed. + */ +STATIC consensus_cache_t * +cdm_cache_get(void) +{ + if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) { + cdm_cache_init(); + } + return cons_diff_cache; +} + +/** + * Helper: given a list of labels, prepend the hex-encoded SHA3 digest + * of the <b>bodylen</b>-byte object at <b>body</b> to those labels, + * with <b>label</b> as its label. + */ +static void +cdm_labels_prepend_sha3(config_line_t **labels, + const char *label, + const uint8_t *body, + size_t bodylen) +{ + uint8_t sha3_digest[DIGEST256_LEN]; + char hexdigest[HEX_DIGEST256_LEN+1]; + crypto_digest256((char *)sha3_digest, + (const char *)body, bodylen, DIGEST_SHA3_256); + base16_encode(hexdigest, sizeof(hexdigest), + (const char *)sha3_digest, sizeof(sha3_digest)); + + config_line_prepend(labels, label, hexdigest); +} + +/** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the + * given label, set <b>digest_out</b> to that value (decoded), and return 0. + * + * Return -1 if there is no such label, and -2 if it is badly formatted. */ +STATIC int +cdm_entry_get_sha3_value(uint8_t *digest_out, + consensus_cache_entry_t *ent, + const char *label) +{ + if (ent == NULL) + return -1; + + const char *hex = consensus_cache_entry_get_value(ent, label); + if (hex == NULL) + return -1; + + int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex)); + if (n != DIGEST256_LEN) + return -2; + else + return 0; +} + +/** + * Helper: look for a consensus with the given <b>flavor</b> and + * <b>valid_after</b> time in the cache. Return that consensus if it's + * present, or NULL if it's missing. + */ +STATIC consensus_cache_entry_t * +cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after) +{ + char formatted_time[ISO_TIME_LEN+1]; + format_iso_time_nospace(formatted_time, valid_after); + const char *flavname = networkstatus_get_flavor_name(flavor); + + /* We'll filter by valid-after time first, since that should + * match the fewest documents. */ + /* We could add an extra hashtable here, but since we only do this scan + * when adding a new consensus, it probably doesn't matter much. */ + smartlist_t *matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_VALID_AFTER, formatted_time); + consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + consensus_cache_entry_t *result = NULL; + if (smartlist_len(matches)) { + result = smartlist_get(matches, 0); + } + smartlist_free(matches); + + return result; +} + +/** Return the maximum age (in seconds) of consensuses that we should consider + * storing. The available space in the directory may impose additional limits + * on how much we store. */ +static int32_t +get_max_age_to_cache(void) +{ + /* The parameter is in hours. */ + const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192; + const int32_t MIN_MAX_AGE_TO_CACHE = 0; + const int32_t MAX_MAX_AGE_TO_CACHE = 8192; + const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff"; + + return 3600 * networkstatus_get_param(NULL, + MAX_AGE_TO_CACHE_NAME, + DEFAULT_MAX_AGE_TO_CACHE, + MIN_MAX_AGE_TO_CACHE, + MAX_MAX_AGE_TO_CACHE); +} + +/** + * Given a string containing a networkstatus consensus, and the results of + * having parsed that consensus, add that consensus to the cache if it is not + * already present and not too old. Create new consensus diffs from or to + * that consensus as appropriate. + * + * Return 0 on success and -1 on failure. + */ +int +consdiffmgr_add_consensus(const char *consensus, + const networkstatus_t *as_parsed) +{ + if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) + return -1; // LCOV_EXCL_LINE + if (BUG(as_parsed->type != NS_TYPE_CONSENSUS)) + return -1; // LCOV_EXCL_LINE + + const consensus_flavor_t flavor = as_parsed->flavor; + const time_t valid_after = as_parsed->valid_after; + + if (valid_after < approx_time() - get_max_age_to_cache()) { + log_info(LD_DIRSERV, "We don't care about this consensus document; it's " + "too old."); + return -1; + } + + /* Do we already have this one? */ + consensus_cache_entry_t *entry = + cdm_cache_lookup_consensus(flavor, valid_after); + if (entry) { + log_info(LD_DIRSERV, "We already have a copy of that consensus"); + return -1; + } + + /* We don't have it. Add it to the cache. */ + return consensus_queue_compression_work(consensus, as_parsed); +} + +/** + * Helper: used to sort two smartlists of consensus_cache_entry_t by their + * LABEL_VALID_AFTER labels. + */ +static int +compare_by_valid_after_(const void **a, const void **b) +{ + const consensus_cache_entry_t *e1 = *a; + const consensus_cache_entry_t *e2 = *b; + /* We're in luck here: sorting UTC iso-encoded values lexically will work + * fine (until 9999). */ + return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER), + consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER)); +} + +/** + * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent + * entry. + */ +static consensus_cache_entry_t * +sort_and_find_most_recent(smartlist_t *lst) +{ + smartlist_sort(lst, compare_by_valid_after_); + if (smartlist_len(lst)) { + return smartlist_get(lst, smartlist_len(lst) - 1); + } else { + return NULL; + } +} + +/** Return i such that compress_consensus_with[i] == method. Return + * -1 if no such i exists. */ +static int +consensus_compression_method_pos(compress_method_t method) +{ + unsigned i; + for (i = 0; i < n_consensus_compression_methods(); ++i) { + if (compress_consensus_with[i] == method) { + return i; + } + } + return -1; +} + +/** + * If we know a consensus with the flavor <b>flavor</b> compressed with + * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as + * for consdiffmgr_find_diff_from(). + */ +consdiff_status_t +consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method) +{ + tor_assert(entry_out); + tor_assert((int)flavor < N_CONSENSUS_FLAVORS); + + int pos = consensus_compression_method_pos(method); + if (pos < 0) { + // We don't compress consensuses with this method. + return CONSDIFF_NOT_FOUND; + } + consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos]; + if (!handle) + return CONSDIFF_NOT_FOUND; + *entry_out = consensus_cache_entry_handle_get(handle); + if (*entry_out) + return CONSDIFF_AVAILABLE; + else + return CONSDIFF_NOT_FOUND; +} + +/** + * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>, + * from the source consensus with the specified digest (which must be SHA3). + * + * If the diff is present, store it into *<b>entry_out</b> and return + * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or + * CONSDIFF_IN_PROGRESS. + */ +consdiff_status_t +consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + int digest_type, + const uint8_t *digest, + size_t digestlen, + compress_method_t method) +{ + if (BUG(digest_type != DIGEST_SHA3_256) || + BUG(digestlen != DIGEST256_LEN)) { + return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE + } + + // Try to look up the entry in the hashtable. + cdm_diff_t search, *ent; + memset(&search, 0, sizeof(search)); + search.flavor = flavor; + search.compress_method = method; + memcpy(search.from_sha3, digest, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + + if (ent == NULL || + ent->cdm_diff_status == CDM_DIFF_ERROR) { + return CONSDIFF_NOT_FOUND; + } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) { + return CONSDIFF_IN_PROGRESS; + } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) { + return CONSDIFF_IN_PROGRESS; + } + + *entry_out = consensus_cache_entry_handle_get(ent->entry); + return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND; + +#if 0 + // XXXX Remove this. I'm keeping it around for now in case we need to + // XXXX debug issues in the hashtable. + char hex[HEX_DIGEST256_LEN+1]; + base16_encode(hex, sizeof(hex), (const char *)digest, digestlen); + const char *flavname = networkstatus_get_flavor_name(flavor); + + smartlist_t *matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FROM_SHA3_DIGEST, hex); + consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + + *entry_out = sort_and_find_most_recent(matches); + consdiff_status_t result = + (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND; + smartlist_free(matches); + + return result; +#endif +} + +/** + * Perform periodic cleanup tasks on the consensus diff cache. Return + * the number of objects marked for deletion. + */ +int +consdiffmgr_cleanup(void) +{ + smartlist_t *objects = smartlist_new(); + smartlist_t *consensuses = smartlist_new(); + smartlist_t *diffs = smartlist_new(); + int n_to_delete = 0; + + log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove"); + + // 1. Delete any consensus or diff or anything whose valid_after is too old. + const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache(); + + consensus_cache_find_all(objects, cdm_cache_get(), + NULL, NULL); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + const char *lv_valid_after = + consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER); + if (! lv_valid_after) { + log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label", + LABEL_VALID_AFTER); + continue; + } + time_t valid_after = 0; + if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) { + log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was " + "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after)); + continue; + } + if (valid_after < valid_after_cutoff) { + log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was " + "too old", LABEL_VALID_AFTER, lv_valid_after); + consensus_cache_entry_mark_for_removal(ent); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(ent); + + // 2. Delete all diffs that lead to a consensus whose valid-after is not the + // latest. + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + /* Determine the most recent consensus of this flavor */ + consensus_cache_find_all(consensuses, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname); + consensus_cache_entry_t *most_recent = + sort_and_find_most_recent(consensuses); + if (most_recent == NULL) + continue; + const char *most_recent_sha3 = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(most_recent_sha3 == NULL)) + continue; // LCOV_EXCL_LINE + + /* consider all such-flavored diffs, and look to see if they match. */ + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *this_diff_target_sha3 = + consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST); + if (!this_diff_target_sha3) + continue; + if (strcmp(this_diff_target_sha3, most_recent_sha3)) { + consensus_cache_entry_mark_for_removal(diff); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(diff); + smartlist_clear(consensuses); + smartlist_clear(diffs); + } + + // 3. Delete all consensuses except the most recent that are compressed with + // an un-preferred method. + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + /* Determine the most recent consensus of this flavor */ + consensus_cache_find_all(consensuses, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname); + consensus_cache_entry_t *most_recent = + sort_and_find_most_recent(consensuses); + if (most_recent == NULL) + continue; + const char *most_recent_sha3_uncompressed = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + const char *retain_methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + + if (BUG(most_recent_sha3_uncompressed == NULL)) + continue; + SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) { + const char *lv_sha3_uncompressed = + consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(! lv_sha3_uncompressed)) + continue; + if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed)) + continue; // This _is_ the most recent. + const char *lv_methodname = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) { + consensus_cache_entry_mark_for_removal(ent); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(ent); + } + + smartlist_free(objects); + smartlist_free(consensuses); + smartlist_free(diffs); + + // Actually remove files, if they're not used. + consensus_cache_delete_pending(cdm_cache_get(), 0); + return n_to_delete; +} + +/** + * Initialize the consensus diff manager and its cache, and configure + * its parameters based on the latest torrc and networkstatus parameters. + */ +void +consdiffmgr_configure(const consdiff_cfg_t *cfg) +{ + if (cfg) + memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg)); + + (void) cdm_cache_get(); +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that the consensus diff manager will need. + */ +int +consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg) +{ + return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg); +} + +/** + * Scan the consensus diff manager's cache for any grossly malformed entries, + * and mark them as deletable. Return 0 if no problems were found; 1 + * if problems were found and fixed. + */ +int +consdiffmgr_validate(void) +{ + /* Right now, we only check for entries that have bad sha3 values */ + int problems = 0; + + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cdm_cache_get(), + NULL, NULL); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) { + uint8_t sha3_expected[DIGEST256_LEN]; + uint8_t sha3_received[DIGEST256_LEN]; + int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST); + if (r == -1) { + /* digest isn't there; that's allowed */ + continue; + } else if (r == -2) { + /* digest is malformed; that's not allowed */ + problems = 1; + consensus_cache_entry_mark_for_removal(obj); + continue; + } + const uint8_t *body; + size_t bodylen; + consensus_cache_entry_incref(obj); + r = consensus_cache_entry_get_body(obj, &body, &bodylen); + if (r == 0) { + crypto_digest256((char *)sha3_received, (const char *)body, bodylen, + DIGEST_SHA3_256); + } + consensus_cache_entry_decref(obj); + if (r < 0) + continue; + + // Deconfuse coverity about the possibility of sha3_received being + // uninitialized + tor_assert(r <= 0); + + if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) { + problems = 1; + consensus_cache_entry_mark_for_removal(obj); + continue; + } + + } SMARTLIST_FOREACH_END(obj); + smartlist_free(objects); + return problems; +} + +/** + * Helper: build new diffs of <b>flavor</b> as needed + */ +static void +consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) +{ + smartlist_t *matches = NULL; + smartlist_t *diffs = NULL; + smartlist_t *compute_diffs_from = NULL; + strmap_t *have_diff_from = NULL; + + // look for the most recent consensus, and for all previous in-range + // consensuses. Do they all have diffs to it? + const char *flavname = networkstatus_get_flavor_name(flavor); + + // 1. find the most recent consensus, and the ones that we might want + // to diff to it. + const char *methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + + matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname); + consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); + if (!most_recent) { + log_info(LD_DIRSERV, "No 'most recent' %s consensus found; " + "not making diffs", flavname); + goto done; + } + tor_assert(smartlist_len(matches)); + smartlist_del(matches, smartlist_len(matches) - 1); + + const char *most_recent_valid_after = + consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER); + if (BUG(most_recent_valid_after == NULL)) + goto done; //LCOV_EXCL_LINE + uint8_t most_recent_sha3[DIGEST256_LEN]; + if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0)) + goto done; //LCOV_EXCL_LINE + + // 2. Find all the relevant diffs _to_ this consensus. These are ones + // that we don't need to compute. + diffs = smartlist_new(); + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_VALID_AFTER, most_recent_valid_after); + consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname); + have_diff_from = strmap_new(); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *va = consensus_cache_entry_get_value(diff, + LABEL_FROM_VALID_AFTER); + if (BUG(va == NULL)) + continue; // LCOV_EXCL_LINE + strmap_set(have_diff_from, va, diff); + } SMARTLIST_FOREACH_END(diff); + + // 3. See which consensuses in 'matches' don't have diffs yet. + smartlist_reverse(matches); // from newest to oldest. + compute_diffs_from = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) { + const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER); + if (BUG(va == NULL)) + continue; // LCOV_EXCL_LINE + if (strmap_get(have_diff_from, va) != NULL) + continue; /* we already have this one. */ + smartlist_add(compute_diffs_from, ent); + /* Since we are not going to serve this as the most recent consensus + * any more, we should stop keeping it mmap'd when it's not in use. + */ + consensus_cache_entry_mark_for_aggressive_release(ent); + } SMARTLIST_FOREACH_END(ent); + + log_info(LD_DIRSERV, + "The most recent %s consensus is valid-after %s. We have diffs to " + "this consensus for %d/%d older %s consensuses. Generating diffs " + "for the other %d.", + flavname, + most_recent_valid_after, + smartlist_len(matches) - smartlist_len(compute_diffs_from), + smartlist_len(matches), + flavname, + smartlist_len(compute_diffs_from)); + + // 4. Update the hashtable; remove entries in this flavor to other + // target consensuses. + cdm_diff_ht_purge(flavor, most_recent_sha3); + + // 5. Actually launch the requests. + SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) { + if (BUG(c == most_recent)) + continue; // LCOV_EXCL_LINE + + uint8_t this_sha3[DIGEST256_LEN]; + if (cdm_entry_get_sha3_value(this_sha3, c, + LABEL_SHA3_DIGEST_AS_SIGNED)<0) { + // Not actually a bug, since we might be running with a directory + // with stale files from before the #22143 fixes. + continue; + } + if (cdm_diff_ht_check_and_note_pending(flavor, + this_sha3, most_recent_sha3)) { + // This is already pending, or we encountered an error. + continue; + } + consensus_diff_queue_diff_work(c, most_recent); + } SMARTLIST_FOREACH_END(c); + + done: + smartlist_free(matches); + smartlist_free(diffs); + smartlist_free(compute_diffs_from); + strmap_free(have_diff_from, NULL); +} + +/** + * Scan the cache for the latest consensuses and add their handles to + * latest_consensus + */ +static void +consdiffmgr_consensus_load(void) +{ + smartlist_t *matches = smartlist_new(); + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + smartlist_clear(matches); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); + if (! most_recent) + continue; // no consensuses. + const char *most_recent_sha3 = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(most_recent_sha3 == NULL)) + continue; // LCOV_EXCL_LINE + consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED, + most_recent_sha3); + + // Everything that remains matches the most recent consensus of this + // flavor. + SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) { + const char *lv_compression = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + compress_method_t method = + compression_method_get_by_name(lv_compression); + int pos = consensus_compression_method_pos(method); + if (pos < 0) + continue; + consensus_cache_entry_handle_free(latest_consensus[flav][pos]); + latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent); + } SMARTLIST_FOREACH_END(ent); + } + smartlist_free(matches); +} + +/** + * Scan the cache for diffs, and add them to the hashtable. + */ +static void +consdiffmgr_diffs_load(void) +{ + smartlist_t *diffs = smartlist_new(); + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *lv_flavor = + consensus_cache_entry_get_value(diff, LABEL_FLAVOR); + if (!lv_flavor) + continue; + int flavor = networkstatus_parse_flavor_name(lv_flavor); + if (flavor < 0) + continue; + const char *lv_compression = + consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE); + compress_method_t method = NO_METHOD; + if (lv_compression) { + method = compression_method_get_by_name(lv_compression); + if (method == UNKNOWN_METHOD) { + continue; + } + } + + uint8_t from_sha3[DIGEST256_LEN]; + uint8_t to_sha3[DIGEST256_LEN]; + if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0) + continue; + if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0) + continue; + + cdm_diff_ht_set_status(flavor, from_sha3, to_sha3, + method, + CDM_DIFF_PRESENT, + consensus_cache_entry_handle_new(diff)); + } SMARTLIST_FOREACH_END(diff); + smartlist_free(diffs); +} + +/** + * Build new diffs as needed. + */ +void +consdiffmgr_rescan(void) +{ + if (cdm_cache_dirty == 0) + return; + + // Clean up here to make room for new diffs, and to ensure that older + // consensuses do not have any entries. + consdiffmgr_cleanup(); + + if (cdm_cache_loaded == 0) { + consdiffmgr_diffs_load(); + consdiffmgr_consensus_load(); + cdm_cache_loaded = 1; + } + + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + consdiffmgr_rescan_flavor_((consensus_flavor_t) flav); + } + + cdm_cache_dirty = 0; +} + +/** + * Helper: compare two files by their from-valid-after and valid-after labels, + * trying to sort in ascending order by from-valid-after (when present) and + * valid-after (when not). Place everything that has neither label first in + * the list. + */ +static int +compare_by_staleness_(const void **a, const void **b) +{ + const consensus_cache_entry_t *e1 = *a; + const consensus_cache_entry_t *e2 = *b; + const char *va1, *fva1, *va2, *fva2; + va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER); + va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER); + fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER); + fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER); + + if (fva1) + va1 = fva1; + if (fva2) + va2 = fva2; + + /* See note about iso-encoded values in compare_by_valid_after_. Also note + * that missing dates will get placed first. */ + return strcmp_opt(va1, va2); +} + +/** If there are not enough unused filenames to store <b>n</b> files, then + * delete old consensuses until there are. (We have to keep track of the + * number of filenames because of the way that the seccomp2 cache works.) + * + * Return 0 on success, -1 on failure. + **/ +static int +consdiffmgr_ensure_space_for_files(int n) +{ + consensus_cache_t *cache = cdm_cache_get(); + if (consensus_cache_get_n_filenames_available(cache) >= n) { + // there are already enough unused filenames. + return 0; + } + // Try a cheap deletion of stuff that's waiting to get deleted. + consensus_cache_delete_pending(cache, 0); + if (consensus_cache_get_n_filenames_available(cache) >= n) { + // okay, _that_ made enough filenames available. + return 0; + } + // Let's get more assertive: clean out unused stuff, and force-remove + // the files. + consdiffmgr_cleanup(); + consensus_cache_delete_pending(cache, 1); + const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache); + if (n_to_remove <= 0) { + // okay, finally! + return 0; + } + + // At this point, we're going to have to throw out objects that will be + // missed. Too bad! + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cache, NULL, NULL); + smartlist_sort(objects, compare_by_staleness_); + int n_marked = 0; + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + consensus_cache_entry_mark_for_removal(ent); + if (++n_marked >= n_to_remove) + break; + } SMARTLIST_FOREACH_END(ent); + + consensus_cache_delete_pending(cache, 1); + if (BUG(n_marked < n_to_remove)) + return -1; + else + return 0; +} + +/** + * Set consensus cache flags on the objects in this consdiffmgr. + */ +static void +consdiffmgr_set_cache_flags(void) +{ + /* Right now, we just mark the consensus objects for aggressive release, + * so that they get mmapped for as little time as possible. */ + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE, + DOCTYPE_CONSENSUS); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + consensus_cache_entry_mark_for_aggressive_release(ent); + } SMARTLIST_FOREACH_END(ent); + smartlist_free(objects); +} + +/** + * Called before shutdown: drop all storage held by the consdiffmgr.c module. + */ +void +consdiffmgr_free_all(void) +{ + cdm_diff_t **diff, **next; + for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) { + cdm_diff_t *this = *diff; + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + } + int i; + unsigned j; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + for (j = 0; j < n_consensus_compression_methods(); ++j) { + consensus_cache_entry_handle_free(latest_consensus[i][j]); + } + } + memset(latest_consensus, 0, sizeof(latest_consensus)); + consensus_cache_free(cons_diff_cache); + cons_diff_cache = NULL; +} + +/* ===== + Thread workers + =====*/ + +typedef struct compressed_result_t { + config_line_t *labels; + /** + * Output: Body of the diff, as compressed. + */ + uint8_t *body; + /** + * Output: length of body_out + */ + size_t bodylen; +} compressed_result_t; + +/** + * Compress the bytestring <b>input</b> of length <b>len</b> using the + * <n>n_methods</b> compression methods listed in the array <b>methods</b>. + * + * For each successful compression, set the fields in the <b>results_out</b> + * array in the position corresponding to the compression method. Use + * <b>labels_in</b> as a basis for the labels of the result. + * + * Return 0 if all compression succeeded; -1 if any failed. + */ +static int +compress_multiple(compressed_result_t *results_out, int n_methods, + const compress_method_t *methods, + const uint8_t *input, size_t len, + const config_line_t *labels_in) +{ + int rv = 0; + int i; + for (i = 0; i < n_methods; ++i) { + compress_method_t method = methods[i]; + const char *methodname = compression_method_get_name(method); + char *result; + size_t sz; + if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) { + results_out[i].body = (uint8_t*)result; + results_out[i].bodylen = sz; + results_out[i].labels = config_lines_dup(labels_in); + cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST, + results_out[i].body, + results_out[i].bodylen); + config_line_prepend(&results_out[i].labels, + LABEL_COMPRESSION_TYPE, + methodname); + } else { + rv = -1; + } + } + return rv; +} + +/** + * Given an array of <b>n</b> compressed_result_t in <b>results</b>, + * as produced by compress_multiple, store them all into the + * consdiffmgr, and store handles to them in the <b>handles_out</b> + * array. + * + * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none + * was stored. + */ +static cdm_diff_status_t +store_multiple(consensus_cache_entry_handle_t **handles_out, + int n, + const compress_method_t *methods, + const compressed_result_t *results, + const char *description) +{ + cdm_diff_status_t status = CDM_DIFF_ERROR; + consdiffmgr_ensure_space_for_files(n); + + int i; + for (i = 0; i < n; ++i) { + compress_method_t method = methods[i]; + uint8_t *body_out = results[i].body; + size_t bodylen_out = results[i].bodylen; + config_line_t *labels = results[i].labels; + const char *methodname = compression_method_get_name(method); + if (body_out && bodylen_out && labels) { + /* Success! Store the results */ + log_info(LD_DIRSERV, "Adding %s, compressed with %s", + description, methodname); + + consensus_cache_entry_t *ent = + consensus_cache_add(cdm_cache_get(), + labels, + body_out, + bodylen_out); + if (BUG(ent == NULL)) + continue; + + status = CDM_DIFF_PRESENT; + handles_out[i] = consensus_cache_entry_handle_new(ent); + consensus_cache_entry_decref(ent); + } + } + return status; +} + +/** + * An object passed to a worker thread that will try to produce a consensus + * diff. + */ +typedef struct consensus_diff_worker_job_t { + /** + * Input: The consensus to compute the diff from. Holds a reference to the + * cache entry, which must not be released until the job is passed back to + * the main thread. The body must be mapped into memory in the main thread. + */ + consensus_cache_entry_t *diff_from; + /** + * Input: The consensus to compute the diff to. Holds a reference to the + * cache entry, which must not be released until the job is passed back to + * the main thread. The body must be mapped into memory in the main thread. + */ + consensus_cache_entry_t *diff_to; + + /** Output: labels and bodies */ + compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)]; +} consensus_diff_worker_job_t; + +/** Given a consensus_cache_entry_t, check whether it has a label claiming + * that it was compressed. If so, uncompress its contents into <b>out</b> and + * set <b>outlen</b> to hold their size. If not, just copy the body into + * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success, + * -1 on failure. + * + * In all cases, the output is nul-terminated. */ +STATIC int +uncompress_or_copy(char **out, size_t *outlen, + consensus_cache_entry_t *ent) +{ + const uint8_t *body; + size_t bodylen; + + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) + return -1; + + const char *lv_compression = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + compress_method_t method = NO_METHOD; + + if (lv_compression) + method = compression_method_get_by_name(lv_compression); + + return tor_uncompress(out, outlen, (const char *)body, bodylen, + method, 1, LOG_WARN); +} + +/** + * Worker function. This function runs inside a worker thread and receives + * a consensus_diff_worker_job_t as its input. + */ +static workqueue_reply_t +consensus_diff_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + consensus_diff_worker_job_t *job = work_; + const uint8_t *diff_from, *diff_to; + size_t len_from, len_to; + int r; + /* We need to have the body already mapped into RAM here. + */ + r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from); + if (BUG(r < 0)) + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to); + if (BUG(r < 0)) + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + + const char *lv_to_valid_after = + consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER); + const char *lv_to_fresh_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL); + const char *lv_to_valid_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL); + const char *lv_to_signatories = + consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES); + const char *lv_from_valid_after = + consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER); + const char *lv_from_digest = + consensus_cache_entry_get_value(job->diff_from, + LABEL_SHA3_DIGEST_AS_SIGNED); + const char *lv_from_flavor = + consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR); + const char *lv_to_flavor = + consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR); + const char *lv_to_digest = + consensus_cache_entry_get_value(job->diff_to, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + + if (! lv_from_digest) { + /* This isn't a bug right now, since it can happen if you're migrating + * from an older version of master to a newer one. The older ones didn't + * annotate their stored consensus objects with sha3-digest-as-signed. + */ + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + + /* All these values are mandatory on the input */ + if (BUG(!lv_to_valid_after) || + BUG(!lv_from_valid_after) || + BUG(!lv_from_flavor) || + BUG(!lv_to_flavor)) { + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + /* The flavors need to match */ + if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) { + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + + char *consensus_diff; + { + char *diff_from_nt = NULL, *diff_to_nt = NULL; + size_t diff_from_nt_len, diff_to_nt_len; + + if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, + job->diff_from) < 0) { + return WQ_RPL_REPLY; + } + if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, + job->diff_to) < 0) { + tor_free(diff_from_nt); + return WQ_RPL_REPLY; + } + tor_assert(diff_from_nt); + tor_assert(diff_to_nt); + + // XXXX ugh; this is going to calculate the SHA3 of both its + // XXXX inputs again, even though we already have that. Maybe it's time + // XXXX to change the API here? + consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); + tor_free(diff_from_nt); + tor_free(diff_to_nt); + } + if (!consensus_diff) { + /* Couldn't generate consensus; we'll leave the reply blank. */ + return WQ_RPL_REPLY; + } + + /* Compress the results and send the reply */ + tor_assert(compress_diffs_with[0] == NO_METHOD); + size_t difflen = strlen(consensus_diff); + job->out[0].body = (uint8_t *) consensus_diff; + job->out[0].bodylen = difflen; + + config_line_t *common_labels = NULL; + if (lv_to_valid_until) + config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until); + if (lv_to_fresh_until) + config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until); + if (lv_to_signatories) + config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories); + cdm_labels_prepend_sha3(&common_labels, + LABEL_SHA3_DIGEST_UNCOMPRESSED, + job->out[0].body, + job->out[0].bodylen); + config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER, + lv_from_valid_after); + config_line_prepend(&common_labels, LABEL_VALID_AFTER, + lv_to_valid_after); + config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor); + config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST, + lv_from_digest); + config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST, + lv_to_digest); + config_line_prepend(&common_labels, LABEL_DOCTYPE, + DOCTYPE_CONSENSUS_DIFF); + + job->out[0].labels = config_lines_dup(common_labels); + cdm_labels_prepend_sha3(&job->out[0].labels, + LABEL_SHA3_DIGEST, + job->out[0].body, + job->out[0].bodylen); + + compress_multiple(job->out+1, + n_diff_compression_methods()-1, + compress_diffs_with+1, + (const uint8_t*)consensus_diff, difflen, common_labels); + + config_free_lines(common_labels); + return WQ_RPL_REPLY; +} + +/** + * Helper: release all storage held in <b>job</b>. + */ +static void +consensus_diff_worker_job_free(consensus_diff_worker_job_t *job) +{ + if (!job) + return; + unsigned u; + for (u = 0; u < n_diff_compression_methods(); ++u) { + config_free_lines(job->out[u].labels); + tor_free(job->out[u].body); + } + consensus_cache_entry_decref(job->diff_from); + consensus_cache_entry_decref(job->diff_to); + tor_free(job); +} + +/** + * Worker function: This function runs in the main thread, and receives + * a consensus_diff_worker_job_t that the worker thread has already + * processed. + */ +static void +consensus_diff_worker_replyfn(void *work_) +{ + tor_assert(in_main_thread()); + tor_assert(work_); + + consensus_diff_worker_job_t *job = work_; + + const char *lv_from_digest = + consensus_cache_entry_get_value(job->diff_from, + LABEL_SHA3_DIGEST_AS_SIGNED); + const char *lv_to_digest = + consensus_cache_entry_get_value(job->diff_to, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + const char *lv_flavor = + consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR); + if (BUG(lv_from_digest == NULL)) + lv_from_digest = "???"; // LCOV_EXCL_LINE + if (BUG(lv_to_digest == NULL)) + lv_to_digest = "???"; // LCOV_EXCL_LINE + + uint8_t from_sha3[DIGEST256_LEN]; + uint8_t to_sha3[DIGEST256_LEN]; + int flav = -1; + int cache = 1; + if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from, + LABEL_SHA3_DIGEST_AS_SIGNED) < 0)) + cache = 0; + if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to, + LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0)) + cache = 0; + if (BUG(lv_flavor == NULL)) { + cache = 0; + } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) { + cache = 0; + } + + consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)]; + memset(handles, 0, sizeof(handles)); + + char description[128]; + tor_snprintf(description, sizeof(description), + "consensus diff from %s to %s", + lv_from_digest, lv_to_digest); + + int status = store_multiple(handles, + n_diff_compression_methods(), + compress_diffs_with, + job->out, + description); + + if (status != CDM_DIFF_PRESENT) { + /* Failure! Nothing to do but complain */ + log_warn(LD_DIRSERV, + "Worker was unable to compute consensus diff " + "from %s to %s", lv_from_digest, lv_to_digest); + /* Cache this error so we don't try to compute this one again. */ + status = CDM_DIFF_ERROR; + } + + unsigned u; + for (u = 0; u < ARRAY_LENGTH(handles); ++u) { + compress_method_t method = compress_diffs_with[u]; + if (cache) { + cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, status, + handles[u]); + } else { + consensus_cache_entry_handle_free(handles[u]); + } + } + + consensus_diff_worker_job_free(job); +} + +/** + * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b> + * in a worker thread. + */ +static int +consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, + consensus_cache_entry_t *diff_to) +{ + tor_assert(in_main_thread()); + + consensus_cache_entry_incref(diff_from); + consensus_cache_entry_incref(diff_to); + + consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->diff_from = diff_from; + job->diff_to = diff_to; + + /* Make sure body is mapped. */ + const uint8_t *body; + size_t bodylen; + int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen); + int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen); + if (r1 < 0 || r2 < 0) + goto err; + + workqueue_entry_t *work; + work = cpuworker_queue_work(consensus_diff_worker_threadfn, + consensus_diff_worker_replyfn, + job); + if (!work) + goto err; + + return 0; + err: + consensus_diff_worker_job_free(job); // includes decrefs. + return -1; +} + +/** + * Holds requests and replies for consensus_compress_workers. + */ +typedef struct consensus_compress_worker_job_t { + char *consensus; + size_t consensus_len; + consensus_flavor_t flavor; + config_line_t *labels_in; + compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)]; +} consensus_compress_worker_job_t; + +/** + * Free all resources held in <b>job</b> + */ +static void +consensus_compress_worker_job_free(consensus_compress_worker_job_t *job) +{ + if (!job) + return; + tor_free(job->consensus); + config_free_lines(job->labels_in); + unsigned u; + for (u = 0; u < n_consensus_compression_methods(); ++u) { + config_free_lines(job->out[u].labels); + tor_free(job->out[u].body); + } + tor_free(job); +} +/** + * Worker function. This function runs inside a worker thread and receives + * a consensus_compress_worker_job_t as its input. + */ +static workqueue_reply_t +consensus_compress_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + consensus_compress_worker_job_t *job = work_; + consensus_flavor_t flavor = job->flavor; + const char *consensus = job->consensus; + size_t bodylen = job->consensus_len; + + config_line_t *labels = config_lines_dup(job->labels_in); + const char *flavname = networkstatus_get_flavor_name(flavor); + + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, + (const uint8_t *)consensus, bodylen); + { + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(consensus, + &start, &end) < 0) { + start = consensus; + end = consensus+bodylen; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, + (const uint8_t *)start, + end - start); + } + config_line_prepend(&labels, LABEL_FLAVOR, flavname); + config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + compress_multiple(job->out, + n_consensus_compression_methods(), + compress_consensus_with, + (const uint8_t*)consensus, bodylen, labels); + config_free_lines(labels); + return WQ_RPL_REPLY; +} + +/** + * Worker function: This function runs in the main thread, and receives + * a consensus_diff_compress_job_t that the worker thread has already + * processed. + */ +static void +consensus_compress_worker_replyfn(void *work_) +{ + consensus_compress_worker_job_t *job = work_; + + consensus_cache_entry_handle_t *handles[ + ARRAY_LENGTH(compress_consensus_with)]; + memset(handles, 0, sizeof(handles)); + + store_multiple(handles, + n_consensus_compression_methods(), + compress_consensus_with, + job->out, + "consensus"); + cdm_cache_dirty = 1; + + unsigned u; + consensus_flavor_t f = job->flavor; + tor_assert((int)f < N_CONSENSUS_FLAVORS); + for (u = 0; u < ARRAY_LENGTH(handles); ++u) { + if (handles[u] == NULL) + continue; + consensus_cache_entry_handle_free(latest_consensus[f][u]); + latest_consensus[f][u] = handles[u]; + } + + consensus_compress_worker_job_free(job); +} + +/** + * If true, we compress in worker threads. + */ +static int background_compression = 0; + +/** + * Queue a job to compress <b>consensus</b> and store its compressed + * text in the cache. + */ +static int +consensus_queue_compression_work(const char *consensus, + const networkstatus_t *as_parsed) +{ + tor_assert(consensus); + tor_assert(as_parsed); + + consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->consensus = tor_strdup(consensus); + job->consensus_len = strlen(consensus); + job->flavor = as_parsed->flavor; + + char va_str[ISO_TIME_LEN+1]; + char vu_str[ISO_TIME_LEN+1]; + char fu_str[ISO_TIME_LEN+1]; + format_iso_time_nospace(va_str, as_parsed->valid_after); + format_iso_time_nospace(fu_str, as_parsed->fresh_until); + format_iso_time_nospace(vu_str, as_parsed->valid_until); + config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str); + config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str); + config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str); + if (as_parsed->voters) { + smartlist_t *hexvoters = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(as_parsed->voters, + networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) == 0) + continue; // didn't sign. + char d[HEX_DIGEST_LEN+1]; + base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN); + smartlist_add_strdup(hexvoters, d); + } SMARTLIST_FOREACH_END(vi); + char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL); + config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers); + tor_free(signers); + SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp)); + smartlist_free(hexvoters); + } + + if (background_compression) { + workqueue_entry_t *work; + work = cpuworker_queue_work(consensus_compress_worker_threadfn, + consensus_compress_worker_replyfn, + job); + if (!work) { + consensus_compress_worker_job_free(job); + return -1; + } + + return 0; + } else { + consensus_compress_worker_threadfn(NULL, job); + consensus_compress_worker_replyfn(job); + return 0; + } +} + +/** + * Tell the consdiffmgr backend to compress consensuses in worker threads. + */ +void +consdiffmgr_enable_background_compression(void) +{ + // This isn't the default behavior because it would break unit tests. + background_compression = 1; +} + +/** Read the set of voters from the cached object <b>ent</b> into + * <b>out</b>, as a list of hex-encoded digests. Return 0 on success, + * -1 if no signatories were recorded. */ +int +consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent, + smartlist_t *out) +{ + tor_assert(ent); + tor_assert(out); + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES); + if (s == NULL) + return -1; + smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0); + return 0; +} + +/** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b> + * and return 0, or return -1 if no such time was recorded. */ +int +consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent, + time_t *out) +{ + tor_assert(ent); + tor_assert(out); + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL); + if (s == NULL || parse_iso_time_nospace(s, out) < 0) + return -1; + else + return 0; +} + +/** Read the valid until timestamp from the cached object <b>ent</b> into + * *<b>out</b> and return 0, or return -1 if no such time was recorded. */ +int +consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent, + time_t *out) +{ + tor_assert(ent); + tor_assert(out); + + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL); + if (s == NULL || parse_iso_time_nospace(s, out) < 0) + return -1; + else + return 0; +} + diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h new file mode 100644 index 0000000000..fe4f9ee239 --- /dev/null +++ b/src/or/consdiffmgr.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONSDIFFMGR_H +#define TOR_CONSDIFFMGR_H + +/** + * Possible outcomes from trying to look up a given consensus diff. + */ +typedef enum consdiff_status_t { + CONSDIFF_AVAILABLE, + CONSDIFF_NOT_FOUND, + CONSDIFF_IN_PROGRESS, +} consdiff_status_t; + +typedef struct consdiff_cfg_t { + int32_t cache_max_num; +} consdiff_cfg_t; + +struct consensus_cache_entry_t; // from conscache.h + +int consdiffmgr_add_consensus(const char *consensus, + const networkstatus_t *as_parsed); + +consdiff_status_t consdiffmgr_find_consensus( + struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method); + +consdiff_status_t consdiffmgr_find_diff_from( + struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + int digest_type, + const uint8_t *digest, + size_t digestlen, + compress_method_t method); + +int consensus_cache_entry_get_voter_id_digests( + const struct consensus_cache_entry_t *ent, + smartlist_t *out); +int consensus_cache_entry_get_fresh_until( + const struct consensus_cache_entry_t *ent, + time_t *out); +int consensus_cache_entry_get_valid_until( + const struct consensus_cache_entry_t *ent, + time_t *out); + +void consdiffmgr_rescan(void); +int consdiffmgr_cleanup(void); +void consdiffmgr_enable_background_compression(void); +void consdiffmgr_configure(const consdiff_cfg_t *cfg); +struct sandbox_cfg_elem; +int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg); +void consdiffmgr_free_all(void); +int consdiffmgr_validate(void); + +#ifdef CONSDIFFMGR_PRIVATE +STATIC unsigned n_diff_compression_methods(void); +STATIC unsigned n_consensus_compression_methods(void); +STATIC consensus_cache_t *cdm_cache_get(void); +STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( + consensus_flavor_t flavor, time_t valid_after); +STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, + consensus_cache_entry_t *ent, + const char *label); +STATIC int uncompress_or_copy(char **out, size_t *outlen, + consensus_cache_entry_t *ent); +#endif + +#endif + diff --git a/src/or/control.c b/src/or/control.c index 04cd8e8dc1..9454a7ab67 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1462,8 +1462,10 @@ handle_control_saveconf(control_connection_t *conn, uint32_t len, const char *body) { (void) len; - (void) body; - if (options_save_current()<0) { + + int force = !strcmpstart(body, "FORCE"); + const or_options_t *options = get_options(); + if ((!force && options->IncludeUsed) || options_save_current() < 0) { connection_write_str_to_buf( "551 Unable to write configuration to disk.\r\n", conn); } else { @@ -1677,6 +1679,8 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup(a); } else if (!strcmp(question, "config-text")) { *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL); + } else if (!strcmp(question, "config-can-saveconf")) { + *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1"); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "dormant")) { @@ -1873,7 +1877,7 @@ getinfo_helper_listeners(control_connection_t *control_conn, /** Implementation helper for GETINFO: knows the answers for questions about * directory information. */ -static int +STATIC int getinfo_helper_dir(control_connection_t *control_conn, const char *question, char **answer, const char **errmsg) @@ -2047,6 +2051,12 @@ getinfo_helper_dir(control_connection_t *control_conn, } } } else if (!strcmp(question, "network-status")) { /* v1 */ + static int network_status_warned = 0; + if (!network_status_warned) { + log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will " + "go away in a future version of Tor."); + network_status_warned = 1; + } routerlist_t *routerlist = router_get_routerlist(); if (!routerlist || !routerlist->routers || list_server_status_v1(routerlist->routers, answer, 1) < 0) { @@ -2824,12 +2834,13 @@ getinfo_helper_events(control_connection_t *control_conn, /** Implementation helper for GETINFO: knows how to enumerate hidden services * created via the control port. */ -static int +STATIC int getinfo_helper_onions(control_connection_t *control_conn, const char *question, char **answer, const char **errmsg) { smartlist_t *onion_list = NULL; + (void) errmsg; /* no errors from this method */ if (control_conn && !strcmp(question, "onions/current")) { onion_list = control_conn->ephemeral_onion_services; @@ -2839,13 +2850,13 @@ getinfo_helper_onions(control_connection_t *control_conn, return 0; } if (!onion_list || smartlist_len(onion_list) == 0) { - if (errmsg) { - *errmsg = "No onion services of the specified type."; + if (answer) { + *answer = tor_strdup(""); } - return -1; - } - if (answer) { + } else { + if (answer) { *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL); + } } return 0; @@ -2924,6 +2935,8 @@ static const getinfo_item_t getinfo_items[] = { ITEM("config-defaults-file", misc, "Current location of the defaults file."), ITEM("config-text", misc, "Return the string that would be written by a saveconf command."), + ITEM("config-can-saveconf", misc, + "Is it possible to save the configuration to the \"torrc\" file?"), ITEM("accounting/bytes", accounting, "Number of bytes read/written so far in the accounting interval."), ITEM("accounting/bytes-left", accounting, @@ -3544,24 +3557,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, } /* Is this a single hop circuit? */ if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) { - const node_t *node = NULL; - char *exit_digest = NULL; - if (circ->build_state && - circ->build_state->chosen_exit && - !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { - exit_digest = circ->build_state->chosen_exit->identity_digest; - node = node_get_by_id(exit_digest); - } - /* Do both the client and relay allow one-hop exit circuits? */ - if (!node || - !node_allows_single_hop_exits(node) || - !get_options()->AllowSingleHopCircuits) { - connection_write_str_to_buf( - "551 Can't attach stream to this one-hop circuit.\r\n", conn); - return 0; - } - tor_assert(exit_digest); - ap_conn->chosen_exit_name = tor_strdup(hex_str(exit_digest, DIGEST_LEN)); + connection_write_str_to_buf( + "551 Can't attach stream to this one-hop circuit.\r\n", conn); + return 0; } if (circ && hop>0) { @@ -6711,8 +6709,8 @@ control_event_bootstrap(bootstrap_status_t status, int progress) * is the connection that caused this problem. */ MOCK_IMPL(void, - control_event_bootstrap_problem, (const char *warn, int reason, - or_connection_t *or_conn)) +control_event_bootstrap_problem, (const char *warn, int reason, + or_connection_t *or_conn)) { int status = bootstrap_percent; const char *tag = "", *summary = ""; @@ -6917,6 +6915,11 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp) goto end; } + /* Without a directory fingerprint at this stage, we can't do much. */ + if (hsdir_fp == NULL) { + goto end; + } + /* OK, we have an onion address so now let's find which descriptor ID * is the one associated with the HSDir fingerprint. */ for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -7006,10 +7009,9 @@ control_event_hs_descriptor_receive_end(const char *action, char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; const char *desc_id = NULL; - if (!action || !id_digest || !rend_data || !onion_address) { - log_warn(LD_BUG, "Called with action==%p, id_digest==%p, " - "rend_data==%p, onion_address==%p", action, id_digest, - rend_data, onion_address); + if (!action || !rend_data || !onion_address) { + log_warn(LD_BUG, "Called with action==%p, rend_data==%p, " + "onion_address==%p", action, rend_data, onion_address); return; } @@ -7032,7 +7034,8 @@ control_event_hs_descriptor_receive_end(const char *action, rend_hsaddress_str_or_unknown(onion_address), rend_auth_type_to_string( TO_REND_DATA_V2(rend_data)->auth_type), - node_describe_longname_by_id(id_digest), + id_digest ? + node_describe_longname_by_id(id_digest) : "UNKNOWN", desc_id_field ? desc_id_field : "", reason_field ? reason_field : ""); @@ -7112,19 +7115,18 @@ control_event_hs_descriptor_uploaded(const char *id_digest, id_digest, NULL); } -/** Send HS_DESC event to inform controller that query <b>rend_query</b> - * failed to retrieve hidden service descriptor identified by - * <b>id_digest</b>. If <b>reason</b> is not NULL, add it to REASON= - * field. +/** Send HS_DESC event to inform controller that query <b>rend_data</b> + * failed to retrieve hidden service descriptor from directory identified by + * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL, + * add it to REASON= field. */ void control_event_hs_descriptor_failed(const rend_data_t *rend_data, const char *id_digest, const char *reason) { - if (!rend_data || !id_digest) { - log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p", - rend_data, id_digest); + if (!rend_data) { + log_warn(LD_BUG, "Called with rend_data==%p", rend_data); return; } control_event_hs_descriptor_receive_end("FAILED", @@ -7132,8 +7134,11 @@ control_event_hs_descriptor_failed(const rend_data_t *rend_data, rend_data, id_digest, reason); } -/** send HS_DESC_CONTENT event after completion of a successful fetch from - * hs directory. */ +/** Send HS_DESC_CONTENT event after completion of a successful fetch from hs + * directory. If <b>hsdir_id_digest</b> is NULL, it is replaced by "UNKNOWN". + * If <b>content</b> is NULL, it is replaced by an empty string. The + * <b>onion_address</b> or <b>desc_id</b> set to NULL will no trigger the + * control event. */ void control_event_hs_descriptor_content(const char *onion_address, const char *desc_id, @@ -7143,9 +7148,9 @@ control_event_hs_descriptor_content(const char *onion_address, static const char *event_name = "HS_DESC_CONTENT"; char *esc_content = NULL; - if (!onion_address || !desc_id || !hsdir_id_digest) { - log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, " - "hsdir_id_digest==%p", onion_address, desc_id, hsdir_id_digest); + if (!onion_address || !desc_id) { + log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ", + onion_address, desc_id); return; } @@ -7160,7 +7165,9 @@ control_event_hs_descriptor_content(const char *onion_address, event_name, rend_hsaddress_str_or_unknown(onion_address), desc_id, - node_describe_longname_by_id(hsdir_id_digest), + hsdir_id_digest ? + node_describe_longname_by_id(hsdir_id_digest) : + "UNKNOWN", esc_content); tor_free(esc_content); } diff --git a/src/or/control.h b/src/or/control.h index 6330c85571..41a194bfcb 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -262,6 +262,11 @@ STATIC crypto_pk_t *add_onion_helper_keyarg(const char *arg, int discard_pk, STATIC rend_authorized_client_t * add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out); +STATIC int getinfo_helper_onions( + control_connection_t *control_conn, + const char *question, + char **answer, + const char **errmsg); STATIC void getinfo_helper_downloads_networkstatus( const char *flavor, download_status_t **dl_to_emit, @@ -285,6 +290,10 @@ STATIC int getinfo_helper_downloads( control_connection_t *control_conn, const char *question, char **answer, const char **errmsg); +STATIC int getinfo_helper_dir( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); #endif diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index fd6de6ea7c..af79fafaa6 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -479,6 +479,20 @@ queue_pending_tasks(void) } } +/** DOCDOC */ +MOCK_IMPL(workqueue_entry_t *, +cpuworker_queue_work,(workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg)) +{ + tor_assert(threadpool); + + return threadpool_queue_work(threadpool, + fn, + reply_fn, + arg); +} + /** Try to tell a cpuworker to perform the public key operations necessary to * respond to <b>onionskin</b> for the circuit <b>circ</b>. * diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h index 62cf0eb164..aedf2fae32 100644 --- a/src/or/cpuworker.h +++ b/src/or/cpuworker.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,6 +14,12 @@ void cpu_init(void); void cpuworkers_rotate_keyinfo(void); +struct workqueue_entry_s; +enum workqueue_reply_t; +MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, ( + enum workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg)); struct create_cell_t; int assign_onionskin_to_cpuworker(or_circuit_t *circ, diff --git a/src/or/dircollate.c b/src/or/dircollate.c index 033a7afe0f..172364c5f5 100644 --- a/src/or/dircollate.c +++ b/src/or/dircollate.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dircollate.h b/src/or/dircollate.h index 358c730cbb..52214282b9 100644 --- a/src/or/dircollate.h +++ b/src/or/dircollate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/directory.c b/src/or/directory.c index 70437fe755..acd694ffee 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRECTORY_PRIVATE @@ -13,7 +13,11 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "conscache.h" +#include "consdiff.h" +#include "consdiffmgr.h" #include "control.h" +#include "compat.h" #define DIRECTORY_PRIVATE #include "directory.h" #include "dirserv.h" @@ -62,7 +66,7 @@ * multi-hop circuits for anonymity. * * Directory requests are launched by calling - * directory_initiate_command_rend() or one of its numerous variants. This + * directory_initiate_request(). This * launch the connection, will construct an HTTP request with * directory_send_command(), send the and wait for a response. The client * later handles the response with connection_dir_client_reached_eof(), @@ -97,9 +101,8 @@ * connection_finished_connecting() in connection.c */ static void directory_send_command(dir_connection_t *conn, - int purpose, int direct, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); + int direct, + const directory_request_t *request); static int body_is_plausible(const char *body, size_t body_len, int purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); @@ -115,21 +118,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int was_descriptor_digests); static void dir_microdesc_download_failed(smartlist_t *failed, int status_code); -static int client_likes_consensus(networkstatus_t *v, const char *want_url); - -static void directory_initiate_command_rend( - const tor_addr_port_t *or_addr_port, - const tor_addr_port_t *dir_addr_port, - const char *digest, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - circuit_guard_state_t *guard_state); +static int client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url); static void connection_dir_close_consensus_fetches( dir_connection_t *except_this_one, const char *resource); @@ -141,6 +131,7 @@ static void connection_dir_close_consensus_fetches( #define ALLOW_DIRECTORY_TIME_SKEW (30*60) #define X_ADDRESS_HEADER "X-Your-Address-Is: " +#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: " /** HTTP cache control: how long do we tell proxies they can cache each * kind of document we serve? */ @@ -420,11 +411,14 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, } else { indirection = DIRIND_DIRECT_CONN; } - directory_initiate_command_routerstatus(rs, dir_purpose, - router_purpose, - indirection, - NULL, payload, upload_len, 0, - NULL); + + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_payload(req, payload, upload_len); + directory_initiate_request(req); + directory_request_free(req); } SMARTLIST_FOREACH_END(ds); if (!found) { char *s = authdir_type_to_string(type); @@ -485,13 +479,94 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, return rs; } +/** + * Set the extra fields in <b>req</b> that are used when requesting a + * consensus of type <b>resource</b>. + * + * Right now, these fields are if-modified-since and x-or-diff-from-consensus. + */ +static void +dir_consensus_request_set_additional_headers(directory_request_t *req, + const char *resource) +{ + time_t if_modified_since = 0; + uint8_t or_diff_from[DIGEST256_LEN]; + int or_diff_from_is_set = 0; + + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ + const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180; + const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72; + const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0; + const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192; + const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] = + "try-diff-for-consensus-newer-than"; + + int flav = FLAV_NS; + if (resource) + flav = networkstatus_parse_flavor_name(resource); + + int32_t max_age_for_diff = 3600 * + networkstatus_get_param(NULL, + TRY_DIFF_FOR_CONSENSUS_NEWER_NAME, + DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER, + MIN_TRY_DIFF_FOR_CONSENSUS_NEWER, + MAX_TRY_DIFF_FOR_CONSENSUS_NEWER); + + if (flav != -1) { + /* IF we have a parsed consensus of this type, we can do an + * if-modified-time based on it. */ + networkstatus_t *v; + v = networkstatus_get_latest_consensus_by_flavor(flav); + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + if (v->valid_after >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } + } else { + /* Otherwise it might be a consensus we don't parse, but which we + * do cache. Look at the cached copy, perhaps. */ + cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ + if (cd) { + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (cd->published >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } + } + + if (if_modified_since > 0) + directory_request_set_if_modified_since(req, if_modified_since); + if (or_diff_from_is_set) { + char hex[HEX_DIGEST256_LEN + 1]; + base16_encode(hex, sizeof(hex), + (const char*)or_diff_from, sizeof(or_diff_from)); + directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex); + } +} + /** Start a connection to a random running directory server, using * connection purpose <b>dir_purpose</b>, intending to fetch descriptors * of purpose <b>router_purpose</b>, and requesting <b>resource</b>. * Use <b>pds_flags</b> as arguments to router_pick_directory_server() * or router_pick_trusteddirserver(). */ -MOCK_IMPL(void, directory_get_from_dirserver, ( +MOCK_IMPL(void, +directory_get_from_dirserver,( uint8_t dir_purpose, uint8_t router_purpose, const char *resource, @@ -506,47 +581,10 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose, resource); dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource); - time_t if_modified_since = 0; if (type == NO_DIRINFO) return; - if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - int flav = FLAV_NS; - networkstatus_t *v; - if (resource) - flav = networkstatus_parse_flavor_name(resource); - - /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus - * period of 1 hour. - */ -#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) - if (flav != -1) { - /* IF we have a parsed consensus of this type, we can do an - * if-modified-time based on it. */ - v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) { - /* In networks with particularly short V3AuthVotingIntervals, - * ask for the consensus if it's been modified since half the - * V3AuthVotingInterval of the most recent consensus. */ - time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; - if (v->fresh_until > v->valid_after - && ims_delay > (v->fresh_until - v->valid_after)/2) { - ims_delay = (v->fresh_until - v->valid_after)/2; - } - if_modified_since = v->valid_after + ims_delay; - } - } else { - /* Otherwise it might be a consensus we don't parse, but which we - * do cache. Look at the cached copy, perhaps. */ - cached_dir_t *cd = dirserv_get_consensus(resource); - /* We have no method of determining the voting interval from an - * unparsed consensus, so we use the default. */ - if (cd) - if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; - } - } - if (!options->FetchServerDescriptors) return; @@ -565,20 +603,20 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( routerinfo_t *ri = node->ri; /* clients always make OR connections to bridges */ tor_addr_port_t or_ap; - tor_addr_port_t nil_dir_ap; + directory_request_t *req = directory_request_new(dir_purpose); /* we are willing to use a non-preferred address if we need to */ fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &or_ap); - tor_addr_make_null(&nil_dir_ap.addr, AF_INET); - nil_dir_ap.port = 0; - directory_initiate_command_rend(&or_ap, - &nil_dir_ap, - ri->cache_info.identity_digest, - dir_purpose, - router_purpose, - DIRIND_ONEHOP, - resource, NULL, 0, if_modified_since, - NULL, guard_state); + directory_request_set_or_addr_port(req, &or_ap); + directory_request_set_directory_id_digest(req, + ri->cache_info.identity_digest); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); } else { if (guard_state) { entry_guard_cancel(&guard_state); @@ -638,12 +676,17 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( if (rs) { const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; - directory_initiate_command_routerstatus(rs, dir_purpose, - router_purpose, - indirection, - resource, NULL, 0, - if_modified_since, - guard_state); + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + if (guard_state) + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); } else { log_notice(LD_DIR, "While fetching directory info, " @@ -669,15 +712,17 @@ directory_get_from_all_authorities(uint8_t dir_purpose, SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(), dir_server_t *, ds) { - routerstatus_t *rs; if (router_digest_is_me(ds->digest)) continue; if (!(ds->type & V3_DIRINFO)) continue; - rs = &ds->fake_status; - directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose, - DIRIND_ONEHOP, resource, NULL, - 0, 0, NULL); + const routerstatus_t *rs = &ds->fake_status; + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); } SMARTLIST_FOREACH_END(ds); } @@ -777,110 +822,6 @@ directory_choose_address_routerstatus(const routerstatus_t *status, return 0; } -/** Same as directory_initiate_command_routerstatus(), but accepts - * rendezvous data to fetch a hidden service descriptor. */ -void -directory_initiate_command_routerstatus_rend(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - circuit_guard_state_t *guard_state) -{ - const or_options_t *options = get_options(); - const node_t *node; - tor_addr_port_t use_or_ap, use_dir_ap; - const int anonymized_connection = dirind_is_anon(indirection); - - tor_assert(status != NULL); - - node = node_get_by_id(status->identity_digest); - - /* XXX The below check is wrong: !node means it's not in the consensus, - * but we haven't checked if we have a descriptor for it -- and also, - * we only care about the descriptor if it's a begindir-style anonymized - * connection. */ - if (!node && anonymized_connection) { - log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " - "don't have its router descriptor.", - routerstatus_describe(status)); - return; - } - - if (options->ExcludeNodes && options->StrictNodes && - routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { - log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but " - "it's in our ExcludedNodes list and StrictNodes is set. " - "Skipping. This choice might make your Tor not work.", - routerstatus_describe(status), - dir_conn_purpose_to_string(dir_purpose)); - return; - } - - /* At this point, if we are a client making a direct connection to a - * directory server, we have selected a server that has at least one address - * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This - * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if - * possible. (If UseBridges is set, clients always use IPv6, and prefer it - * by default.) - * - * Now choose an address that we can use to connect to the directory server. - */ - if (directory_choose_address_routerstatus(status, indirection, &use_or_ap, - &use_dir_ap) < 0) { - return; - } - - /* We don't retry the alternate OR/Dir address for the same directory if - * the address we choose fails (#6772). - * Instead, we'll retry another directory on failure. */ - - directory_initiate_command_rend(&use_or_ap, &use_dir_ap, - status->identity_digest, - dir_purpose, router_purpose, - indirection, resource, - payload, payload_len, if_modified_since, - rend_query, - guard_state); -} - -/** Launch a new connection to the directory server <b>status</b> to - * upload or download a server or rendezvous - * descriptor. <b>dir_purpose</b> determines what - * kind of directory connection we're launching, and must be one of - * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b> - * specifies the descriptor purposes we have in mind (currently only - * used for FETCH_DIR). - * - * When uploading, <b>payload</b> and <b>payload_len</b> determine the content - * of the HTTP post. Otherwise, <b>payload</b> should be NULL. - * - * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we - * want to fetch. - */ -MOCK_IMPL(void, directory_initiate_command_routerstatus, - (const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - circuit_guard_state_t *guard_state)) -{ - directory_initiate_command_routerstatus_rend(status, dir_purpose, - router_purpose, - indirection, resource, - payload, payload_len, - if_modified_since, NULL, - guard_state); -} - /** Return true iff <b>conn</b> is the client side of a directory connection * we launched to ourself in order to determine the reachability of our * dir_port. */ @@ -1064,6 +1005,47 @@ directory_must_use_begindir(const or_options_t *options) return !public_server_mode(options); } +struct directory_request_t { + /** + * These fields specify which directory we're contacting. Routerstatus, + * if present, overrides the other fields. + * + * @{ */ + tor_addr_port_t or_addr_port; + tor_addr_port_t dir_addr_port; + char digest[DIGEST_LEN]; + + const routerstatus_t *routerstatus; + /** @} */ + /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what + * kind of operation we'll be doing (upload/download), and of what kind + * of document. */ + uint8_t dir_purpose; + /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo + * and extrainfo docs. */ + uint8_t router_purpose; + /** Enum: determines whether to anonymize, and whether to use dirport or + * orport. */ + dir_indirection_t indirection; + /** Alias to the variable part of the URL for this request */ + const char *resource; + /** Alias to the payload to upload (if any) */ + const char *payload; + /** Number of bytes to upload from payload</b> */ + size_t payload_len; + /** Value to send in an if-modified-since header, or 0 for none. */ + time_t if_modified_since; + /** Hidden-service-specific information */ + const rend_data_t *rend_query; + /** Extra headers to append to the request */ + config_line_t *additional_headers; + /** */ + /** Used internally to directory.c: gets informed when the attempt to + * connect to the directory succeeds or fails, if that attempt bears on the + * directory's usability as a directory guard. */ + circuit_guard_state_t *guard_state; +}; + /** Evaluate the situation and decide if we should use an encrypted * "begindir-style" connection for this directory request. * 0) If there is no DirPort, yes. @@ -1077,14 +1059,16 @@ directory_must_use_begindir(const or_options_t *options) */ static int directory_command_should_use_begindir(const or_options_t *options, - const tor_addr_t *or_addr, int or_port, - const tor_addr_t *dir_addr, int dir_port, - uint8_t router_purpose, - dir_indirection_t indirection, + const directory_request_t *req, const char **reason) { - (void) router_purpose; - (void) dir_addr; + const tor_addr_t *or_addr = &req->or_addr_port.addr; + //const tor_addr_t *dir_addr = &req->dir_addr_port.addr; + const int or_port = req->or_addr_port.port; + const int dir_port = req->dir_addr_port.port; + + const dir_indirection_t indirection = req->indirection; + tor_assert(reason); *reason = NULL; @@ -1124,68 +1108,290 @@ directory_command_should_use_begindir(const or_options_t *options, return 1; } -/** Helper for directory_initiate_command_rend: send the - * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>, - * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose - * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and - * <b>router_purpose</b>, making an (in)direct connection as specified in - * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of - * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>. +/** + * Create and return a new directory_request_t with purpose + * <b>dir_purpose</b>. + */ +directory_request_t * +directory_request_new(uint8_t dir_purpose) +{ + tor_assert(dir_purpose >= DIR_PURPOSE_MIN_); + tor_assert(dir_purpose <= DIR_PURPOSE_MAX_); + tor_assert(dir_purpose != DIR_PURPOSE_SERVER); + tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2); + + directory_request_t *result = tor_malloc_zero(sizeof(*result)); + tor_addr_make_null(&result->or_addr_port.addr, AF_INET); + result->or_addr_port.port = 0; + tor_addr_make_null(&result->dir_addr_port.addr, AF_INET); + result->dir_addr_port.port = 0; + result->dir_purpose = dir_purpose; + result->router_purpose = ROUTER_PURPOSE_GENERAL; + result->indirection = DIRIND_ONEHOP; + return result; +} +/** + * Release all resources held by <b>req</b>. */ void -directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, - const tor_addr_t *dir_addr, uint16_t dir_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since) +directory_request_free(directory_request_t *req) { - tor_addr_port_t or_ap, dir_ap; + if (req == NULL) + return; + config_free_lines(req->additional_headers); + tor_free(req); +} +/** + * Set the address and OR port to use for this directory request. If there is + * no OR port, we'll have to connect over the dirport. (If there are both, + * the indirection setting determins which to use.) + */ +void +directory_request_set_or_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->or_addr_port, p, sizeof(*p)); +} +/** + * Set the address and dirport to use for this directory request. If there + * is no dirport, we'll have to connect over the OR port. (If there are both, + * the indirection setting determins which to use.) + */ +void +directory_request_set_dir_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->dir_addr_port, p, sizeof(*p)); +} +/** + * Set the RSA identity digest of the directory to use for this directory + * request. + */ +void +directory_request_set_directory_id_digest(directory_request_t *req, + const char *digest) +{ + memcpy(req->digest, digest, DIGEST_LEN); +} +/** + * Set the router purpose associated with uploaded and downloaded router + * descriptors and extrainfo documents in this directory request. The purpose + * must be one of ROUTER_PURPOSE_GENERAL (the default) or + * ROUTER_PURPOSE_BRIDGE. + */ +void +directory_request_set_router_purpose(directory_request_t *req, + uint8_t router_purpose) +{ + tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL || + router_purpose == ROUTER_PURPOSE_BRIDGE); + // assert that it actually makes sense to set this purpose, given + // the dir_purpose. + req->router_purpose = router_purpose; +} +/** + * Set the indirection to be used for the directory request. The indirection + * parameter configures whether to connect to a DirPort or ORPort, and whether + * to anonymize the connection. DIRIND_ONEHOP (use ORPort, don't anonymize) + * is the default. See dir_indirection_t for more information. + */ +void +directory_request_set_indirection(directory_request_t *req, + dir_indirection_t indirection) +{ + req->indirection = indirection; +} - /* Use the null tor_addr and 0 port if the address or port isn't valid. */ - if (tor_addr_port_is_valid(or_addr, or_port, 0)) { - tor_addr_copy(&or_ap.addr, or_addr); - or_ap.port = or_port; - } else { - /* the family doesn't matter here, so make it IPv4 */ - tor_addr_make_null(&or_ap.addr, AF_INET); - or_ap.port = or_port = 0; +/** + * Set a pointer to the resource to request from a directory. Different + * request types use resources to indicate different components of their URL. + * Note that only an alias to <b>resource</b> is stored, so the + * <b>resource</b> must outlive the request. + */ +void +directory_request_set_resource(directory_request_t *req, + const char *resource) +{ + req->resource = resource; +} +/** + * Set a pointer to the payload to include with this directory request, along + * with its length. Note that only an alias to <b>payload</b> is stored, so + * the <b>payload</b> must outlive the request. + */ +void +directory_request_set_payload(directory_request_t *req, + const char *payload, + size_t payload_len) +{ + tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose)); + + req->payload = payload; + req->payload_len = payload_len; +} +/** + * Set an if-modified-since date to send along with the request. The + * default is 0 (meaning, send no if-modified-since header). + */ +void +directory_request_set_if_modified_since(directory_request_t *req, + time_t if_modified_since) +{ + req->if_modified_since = if_modified_since; +} + +/** Include a header of name <b>key</b> with content <b>val</b> in the + * request. Neither may include newlines or other odd characters. Their + * ordering is not currently guaranteed. + * + * Note that, as elsewhere in this module, header keys include a trailing + * colon and space. + */ +void +directory_request_add_header(directory_request_t *req, + const char *key, + const char *val) +{ + config_line_prepend(&req->additional_headers, key, val); +} +/** + * Set an object containing HS data to be associated with this request. Note + * that only an alias to <b>query</b> is stored, so the <b>query</b> object + * must outlive the request. + */ +void +directory_request_set_rend_query(directory_request_t *req, + const rend_data_t *query) +{ + if (query) { + tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 || + req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); } + req->rend_query = query; +} +/** Set a static circuit_guard_state_t object to affliate with the request in + * <b>req</b>. This object will receive notification when the attempt to + * connect to the guard either succeeds or fails. */ +void +directory_request_set_guard_state(directory_request_t *req, + circuit_guard_state_t *state) +{ + req->guard_state = state; +} - if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) { - tor_addr_copy(&dir_ap.addr, dir_addr); - dir_ap.port = dir_port; - } else { - /* the family doesn't matter here, so make it IPv4 */ - tor_addr_make_null(&dir_ap.addr, AF_INET); - dir_ap.port = dir_port = 0; +/** + * Internal: Return true if any information for contacting the directory in + * <b>req</b> has been set, other than by the routerstatus. */ +static int +directory_request_dir_contact_info_specified(const directory_request_t *req) +{ + /* We only check for ports here, since we don't use an addr unless the port + * is set */ + return (req->or_addr_port.port || + req->dir_addr_port.port || + ! tor_digest_is_zero(req->digest)); +} + +/** + * Set the routerstatus to use for the directory associated with this + * request. If this option is set, then no other function to set the + * directory's address or identity should be called. + */ +void +directory_request_set_routerstatus(directory_request_t *req, + const routerstatus_t *status) +{ + req->routerstatus = status; +} +/** + * Helper: update the addresses, ports, and identities in <b>req</b> + * from the routerstatus object in <b>req</b>. Return 0 on success. + * On failure, warn and return -1. + */ +static int +directory_request_set_dir_from_routerstatus(directory_request_t *req) + +{ + const routerstatus_t *status = req->routerstatus; + if (BUG(status == NULL)) + return -1; + const or_options_t *options = get_options(); + const node_t *node; + tor_addr_port_t use_or_ap, use_dir_ap; + const int anonymized_connection = dirind_is_anon(req->indirection); + + tor_assert(status != NULL); + + node = node_get_by_id(status->identity_digest); + + /* XXX The below check is wrong: !node means it's not in the consensus, + * but we haven't checked if we have a descriptor for it -- and also, + * we only care about the descriptor if it's a begindir-style anonymized + * connection. */ + if (!node && anonymized_connection) { + log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " + "don't have its router descriptor.", + routerstatus_describe(status)); + return -1; + } + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { + log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping. This choice might make your Tor not work.", + routerstatus_describe(status), + dir_conn_purpose_to_string(req->dir_purpose)); + return -1; } - directory_initiate_command_rend(&or_ap, &dir_ap, - digest, dir_purpose, - router_purpose, indirection, - resource, payload, payload_len, - if_modified_since, NULL, NULL); + /* At this point, if we are a client making a direct connection to a + * directory server, we have selected a server that has at least one address + * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This + * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if + * possible. (If UseBridges is set, clients always use IPv6, and prefer it + * by default.) + * + * Now choose an address that we can use to connect to the directory server. + */ + if (directory_choose_address_routerstatus(status, + req->indirection, &use_or_ap, + &use_dir_ap) < 0) { + return -1; + } + + directory_request_set_or_addr_port(req, &use_or_ap); + directory_request_set_dir_addr_port(req, &use_dir_ap); + directory_request_set_directory_id_digest(req, status->identity_digest); + return 0; } -/** Same as directory_initiate_command(), but accepts rendezvous data to - * fetch a hidden service descriptor, and takes its address & port arguments - * as tor_addr_port_t. */ -static void -directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, - const tor_addr_port_t *dir_addr_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - circuit_guard_state_t *guard_state) +/** + * Launch the provided directory request, configured in <b>request</b>. + * After this function is called, you can free <b>request</b>. + */ +MOCK_IMPL(void, +directory_initiate_request,(directory_request_t *request)) { - tor_assert(or_addr_port); - tor_assert(dir_addr_port); + tor_assert(request); + if (request->routerstatus) { + tor_assert_nonfatal( + ! directory_request_dir_contact_info_specified(request)); + if (directory_request_set_dir_from_routerstatus(request) < 0) { + return; + } + } + + const tor_addr_port_t *or_addr_port = &request->or_addr_port; + const tor_addr_port_t *dir_addr_port = &request->dir_addr_port; + const char *digest = request->digest; + const uint8_t dir_purpose = request->dir_purpose; + const uint8_t router_purpose = request->router_purpose; + const dir_indirection_t indirection = request->indirection; + const char *resource = request->resource; + const rend_data_t *rend_query = request->rend_query; + circuit_guard_state_t *guard_state = request->guard_state; + tor_assert(or_addr_port->port || dir_addr_port->port); tor_assert(digest); @@ -1195,11 +1401,9 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, const char *begindir_reason = NULL; /* Should the connection be to a relay's OR port (and inside that we will * send our directory request)? */ - const int use_begindir = directory_command_should_use_begindir(options, - &or_addr_port->addr, or_addr_port->port, - &dir_addr_port->addr, dir_addr_port->port, - router_purpose, indirection, - &begindir_reason); + const int use_begindir = + directory_command_should_use_begindir(options, request, &begindir_reason); + /* Will the connection go via a three-hop Tor circuit? Note that this * is separate from whether it will use_begindir. */ const int anonymized_connection = dirind_is_anon(indirection); @@ -1300,9 +1504,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, /* fall through */ case 0: /* queue the command on the outbuf */ - directory_send_command(conn, dir_purpose, 1, resource, - payload, payload_len, - if_modified_since); + directory_send_command(conn, 1, request); connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ @@ -1356,9 +1558,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, } conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; /* queue the command on the outbuf */ - directory_send_command(conn, dir_purpose, 0, resource, - payload, payload_len, - if_modified_since); + directory_send_command(conn, 0, request); connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(ENTRY_TO_CONN(linked_conn)); @@ -1368,7 +1568,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, /** Return true iff anything we say on <b>conn</b> is being encrypted before * we send it to the client/server. */ int -connection_dir_is_encrypted(dir_connection_t *conn) +connection_dir_is_encrypted(const dir_connection_t *conn) { /* Right now it's sufficient to see if conn is or has been linked, since * the only thing it could be linked to is an edge connection on a @@ -1461,15 +1661,23 @@ copy_ipv6_address(char* destination, const char* source, size_t len, } } -/** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command(). +/** Queue an appropriate HTTP command for <b>request</b> on + * <b>conn</b>-\>outbuf. If <b>direct</b> is true, we're making a + * non-anonymized connection to the dirport. */ static void directory_send_command(dir_connection_t *conn, - int purpose, int direct, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since) + const int direct, + const directory_request_t *req) { + tor_assert(req); + const int purpose = req->dir_purpose; + const char *resource = req->resource; + const char *payload = req->payload; + const size_t payload_len = req->payload_len; + const time_t if_modified_since = req->if_modified_since; + const int anonymized_connection = dirind_is_anon(req->indirection); + char proxystring[256]; char hoststring[128]; /* NEEDS to be the same size hoststring. @@ -1477,7 +1685,10 @@ directory_send_command(dir_connection_t *conn, char decorated_address[128]; smartlist_t *headers = smartlist_new(); char *url; + char *accept_encoding; + size_t url_len; char request[8192]; + size_t request_len, total_request_len = 0; const char *httpcommand = NULL; tor_assert(conn); @@ -1531,6 +1742,22 @@ directory_send_command(dir_connection_t *conn, proxystring[0] = 0; } + if (! anonymized_connection) { + /* Add Accept-Encoding. */ + accept_encoding = accept_encoding_header(); + smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n", + accept_encoding); + tor_free(accept_encoding); + } + + /* Add additional headers, if any */ + { + config_line_t *h; + for (h = req->additional_headers; h; h = h->next) { + smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value); + } + } + switch (purpose) { case DIR_PURPOSE_FETCH_CONSENSUS: /* resource is optional. If present, it's a flavor name */ @@ -1623,8 +1850,14 @@ directory_send_command(dir_connection_t *conn, } tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring); - connection_write_to_buf(request, strlen(request), TO_CONN(conn)); - connection_write_to_buf(url, strlen(url), TO_CONN(conn)); + + request_len = strlen(request); + total_request_len += request_len; + connection_write_to_buf(request, request_len, TO_CONN(conn)); + + url_len = strlen(url); + total_request_len += url_len; + connection_write_to_buf(url, url_len, TO_CONN(conn)); tor_free(url); if (!strcmp(httpcommand, "POST") || payload) { @@ -1639,15 +1872,27 @@ directory_send_command(dir_connection_t *conn, tor_free(header); } - connection_write_to_buf(request, strlen(request), TO_CONN(conn)); + request_len = strlen(request); + total_request_len += request_len; + connection_write_to_buf(request, request_len, TO_CONN(conn)); if (payload) { /* then send the payload afterwards too */ connection_write_to_buf(payload, payload_len, TO_CONN(conn)); + total_request_len += payload_len; } SMARTLIST_FOREACH(headers, char *, h, tor_free(h)); smartlist_free(headers); + + log_debug(LD_DIR, + "Sent request to directory server '%s:%d': " + "(purpose: %d, request size: " U64_FORMAT ", " + "payload size: " U64_FORMAT ")", + conn->base_.address, conn->base_.port, + conn->base_.purpose, + U64_PRINTF_ARG(total_request_len), + U64_PRINTF_ARG(payload ? payload_len : 0)); } /** Parse an HTTP request string <b>headers</b> of the form @@ -1832,16 +2077,15 @@ parse_http_response(const char *headers, int *code, time_t *date, if (!strcmpstart(s, "Content-Encoding: ")) { enc = s+18; break; }); - if (!enc || !strcmp(enc, "identity")) { + + if (enc == NULL) *compression = NO_METHOD; - } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) { - *compression = ZLIB_METHOD; - } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) { - *compression = GZIP_METHOD; - } else { - log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", - escaped(enc)); - *compression = UNKNOWN_METHOD; + else { + *compression = compression_method_get_by_name(enc); + + if (*compression == UNKNOWN_METHOD) + log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", + escaped(enc)); } } SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s)); @@ -1914,6 +2158,39 @@ load_downloaded_routers(const char *body, smartlist_t *which, return added; } +/** A structure to hold arguments passed into each directory response + * handler */ +typedef struct response_handler_args_t { + int status_code; + const char *reason; + const char *body; + size_t body_len; + const char *headers; +} response_handler_args_t; + +static int handle_response_fetch_consensus(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_certificate(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_status_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_detached_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_desc(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_microdesc(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_dir(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); + /** We are a client, and we've finished reading the server's * response. Parse it and act appropriately. * @@ -1926,8 +2203,8 @@ load_downloaded_routers(const char *body, smartlist_t *which, static int connection_dir_client_reached_eof(dir_connection_t *conn) { - char *body; - char *headers; + char *body = NULL; + char *headers = NULL; char *reason = NULL; size_t body_len = 0; int status_code; @@ -1936,11 +2213,17 @@ connection_dir_client_reached_eof(dir_connection_t *conn) compress_method_t compression; int plausible; int skewed = 0; + int rv; int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - time_t now = time(NULL); - int src_code; + size_t received_bytes; + const int anonymized_connection = + purpose_needs_anonymity(conn->base_.purpose, + conn->router_purpose, + conn->requested_resource); + + received_bytes = connection_get_inbuf_len(TO_CONN(conn)); switch (connection_fetch_from_buf_http(TO_CONN(conn), &headers, MAX_HEADERS_SIZE, @@ -1962,17 +2245,26 @@ connection_dir_client_reached_eof(dir_connection_t *conn) &compression, &reason) < 0) { log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.", conn->base_.address, conn->base_.port); - tor_free(body); tor_free(headers); - return -1; + + rv = -1; + goto done; } if (!reason) reason = tor_strdup("[no reason given]"); - log_debug(LD_DIR, + tor_log(LOG_DEBUG, LD_DIR, "Received response from directory server '%s:%d': %d %s " - "(purpose: %d)", + "(purpose: %d, response size: " U64_FORMAT +#ifdef MEASUREMENTS_21206 + ", data cells received: %d, data cells sent: %d" +#endif + ", compression: %d)", conn->base_.address, conn->base_.port, status_code, - escaped(reason), - conn->base_.purpose); + escaped(reason), conn->base_.purpose, + U64_PRINTF_ARG(received_bytes), +#ifdef MEASUREMENTS_21206 + conn->data_cells_received, conn->data_cells_sent, +#endif + compression); if (conn->guard_state) { /* we count the connection as successful once we can read from it. We do @@ -2023,13 +2315,14 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. I'll try again soon.", status_code, escaped(reason), conn->base_.address, conn->base_.port); + time_t now = approx_time(); if ((rs = router_get_mutable_consensus_status_by_id(id_digest))) rs->last_dir_503_at = now; if ((ds = router_get_fallback_dirserver_by_digest(id_digest))) ds->fake_status.last_dir_503_at = now; - tor_free(body); tor_free(headers); tor_free(reason); - return -1; + rv = -1; + goto done; } plausible = body_is_plausible(body, body_len, conn->base_.purpose); @@ -2040,48 +2333,60 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (compression == UNKNOWN_METHOD || guessed != compression) { /* Tell the user if we don't believe what we're told about compression.*/ const char *description1, *description2; - if (compression == ZLIB_METHOD) - description1 = "as deflated"; - else if (compression == GZIP_METHOD) - description1 = "as gzipped"; - else if (compression == NO_METHOD) - description1 = "as uncompressed"; - else - description1 = "with an unknown Content-Encoding"; - if (guessed == ZLIB_METHOD) - description2 = "deflated"; - else if (guessed == GZIP_METHOD) - description2 = "gzipped"; - else if (!plausible) + + description1 = compression_method_get_human_name(compression); + + if (BUG(description1 == NULL)) + description1 = compression_method_get_human_name(UNKNOWN_METHOD); + + if (guessed == UNKNOWN_METHOD && !plausible) description2 = "confusing binary junk"; else - description2 = "uncompressed"; + description2 = compression_method_get_human_name(guessed); - log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, " + log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled as %s, " "but it seems to be %s.%s", conn->base_.address, conn->base_.port, description1, description2, (compression>0 && guessed>0)?" Trying both.":""); } - /* Try declared compression first if we can. */ - if (compression == GZIP_METHOD || compression == ZLIB_METHOD) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression, - !allow_partial, LOG_PROTOCOL_WARN); + + /* Try declared compression first if we can. + * tor_compress_supports_method() also returns true for NO_METHOD. + * Ensure that the server is not sending us data compressed using a + * compression method that is not allowed for anonymous connections. */ + if (anonymized_connection && + ! allowed_anonymous_connection_compression_method(compression)) { + warn_disallowed_anonymous_compression_method(compression); + rv = -1; + goto done; + } + + if (tor_compress_supports_method(compression)) + tor_uncompress(&new_body, &new_len, body, body_len, compression, + !allow_partial, LOG_PROTOCOL_WARN); + /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ - if (!new_body && - (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && + if (anonymized_connection && + ! allowed_anonymous_connection_compression_method(guessed)) { + warn_disallowed_anonymous_compression_method(guessed); + rv = -1; + goto done; + } + + if (!new_body && tor_compress_supports_method(guessed) && compression != guessed) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, guessed, + !allow_partial, LOG_PROTOCOL_WARN); /* If we're pretty sure that we have a compressed directory, and * we didn't manage to uncompress it, then warn and bail. */ if (!plausible && !new_body) { log_fn(LOG_PROTOCOL_WARN, LD_HTTP, "Unable to decompress HTTP body (server '%s:%d').", conn->base_.address, conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; + rv = -1; + goto done; } if (new_body) { tor_free(body); @@ -2090,473 +2395,737 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - int r; - const char *flavname = conn->requested_resource; - if (status_code != 200) { - int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; - tor_log(severity, LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching consensus directory.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - networkstatus_consensus_download_failed(status_code, flavname); - return -1; + response_handler_args_t args; + memset(&args, 0, sizeof(args)); + args.status_code = status_code; + args.reason = reason; + args.body = body; + args.body_len = body_len; + args.headers = headers; + + switch (conn->base_.purpose) { + case DIR_PURPOSE_FETCH_CONSENSUS: + rv = handle_response_fetch_consensus(conn, &args); + break; + case DIR_PURPOSE_FETCH_CERTIFICATE: + rv = handle_response_fetch_certificate(conn, &args); + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + rv = handle_response_fetch_status_vote(conn, &args); + break; + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + rv = handle_response_fetch_detached_signatures(conn, &args); + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + rv = handle_response_fetch_desc(conn, &args); + break; + case DIR_PURPOSE_FETCH_MICRODESC: + rv = handle_response_fetch_microdesc(conn, &args); + break; + case DIR_PURPOSE_FETCH_RENDDESC_V2: + rv = handle_response_fetch_renddesc_v2(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_DIR: + rv = handle_response_upload_dir(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_SIGNATURES: + rv = handle_response_upload_signatures(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_VOTE: + rv = handle_response_upload_vote(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + rv = handle_response_upload_renddesc_v2(conn, &args); + break; + default: + tor_assert_nonfatal_unreached(); + rv = -1; + break; + } + + done: + tor_free(body); + tor_free(headers); + tor_free(reason); + return rv; +} + +/** + * Handler function: processes a response to a request for a networkstatus + * consensus document by checking the consensus, storing it, and marking + * router requests as reachable. + **/ +static int +handle_response_fetch_consensus(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS); + const int status_code = args->status_code; + const char *body = args->body; + const size_t body_len = args->body_len; + const char *reason = args->reason; + const time_t now = approx_time(); + + const char *consensus; + char *new_consensus = NULL; + const char *sourcename; + + int r; + const char *flavname = conn->requested_resource; + if (status_code != 200) { + int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; + tor_log(severity, LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching consensus directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + networkstatus_consensus_download_failed(status_code, flavname); + return -1; + } + + if (looks_like_a_consensus_diff(body, body_len)) { + /* First find our previous consensus. Maybe it's in ram, maybe not. */ + cached_dir_t *cd = dirserv_get_consensus(flavname); + const char *consensus_body; + char *owned_consensus = NULL; + if (cd) { + consensus_body = cd->dir; + } else { + owned_consensus = networkstatus_read_cached_consensus(flavname); + consensus_body = owned_consensus; } - log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if ((r=networkstatus_set_current_consensus(body, flavname, 0, - conn->identity_digest))<0) { - log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, - "Unable to load %s consensus directory downloaded from " - "server '%s:%d'. I'll try again soon.", - flavname, conn->base_.address, conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); + if (!consensus_body) { + log_warn(LD_DIR, "Received a consensus diff, but we can't find " + "any %s-flavored consensus in our current cache.",flavname); networkstatus_consensus_download_failed(0, flavname); + // XXXX if this happens too much, see below return -1; } - /* If we launched other fetches for this consensus, cancel them. */ - connection_dir_close_consensus_fetches(conn, flavname); - - /* launches router downloads as needed */ - routers_update_all_from_networkstatus(now, 3); - update_microdescs_from_networkstatus(now); - update_microdesc_downloads(now); - directory_info_has_arrived(now, 0, 0); - if (authdir_mode_v3(get_options())) { - sr_act_post_consensus( - networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); - } - log_info(LD_DIR, "Successfully loaded consensus."); - } - - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) { - if (status_code != 200) { - log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/keys/%s\".", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource); - connection_dir_download_cert_failed(conn, status_code); - tor_free(body); tor_free(headers); tor_free(reason); + new_consensus = consensus_diff_apply(consensus_body, body); + tor_free(owned_consensus); + if (new_consensus == NULL) { + log_warn(LD_DIR, "Could not apply consensus diff received from server " + "'%s:%d'", conn->base_.address, conn->base_.port); + // XXXX If this happens too many times, we should maybe not use + // XXXX this directory for diffs any more? + networkstatus_consensus_download_failed(0, flavname); return -1; } - log_info(LD_DIR,"Received authority certificates (size %d) from server " + log_info(LD_DIR, "Applied consensus diff (size %d) from server " + "'%s:%d', resulting in a new consensus document (size %d).", + (int)body_len, conn->base_.address, conn->base_.port, + (int)strlen(new_consensus)); + consensus = new_consensus; + sourcename = "generated based on a diff"; + } else { + log_info(LD_DIR,"Received consensus directory (body size %d) from server " "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); + consensus = body; + sourcename = "downloaded"; + } + + if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + conn->identity_digest))<0) { + log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, + "Unable to load %s consensus directory %s from " + "server '%s:%d'. I'll try again soon.", + flavname, sourcename, conn->base_.address, conn->base_.port); + networkstatus_consensus_download_failed(0, flavname); + tor_free(new_consensus); + return -1; + } - /* - * Tell trusted_dirs_load_certs_from_string() whether it was by fp - * or fp-sk pair. - */ - src_code = -1; - if (!strcmpstart(conn->requested_resource, "fp/")) { - src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST; - } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) { - src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST; - } + /* If we launched other fetches for this consensus, cancel them. */ + connection_dir_close_consensus_fetches(conn, flavname); - if (src_code != -1) { - if (trusted_dirs_load_certs_from_string(body, src_code, 1, - conn->identity_digest)<0) { - log_warn(LD_DIR, "Unable to parse fetched certificates"); - /* if we fetched more than one and only some failed, the successful - * ones got flushed to disk so it's safe to call this on them */ - connection_dir_download_cert_failed(conn, status_code); - } else { - directory_info_has_arrived(now, 0, 0); - log_info(LD_DIR, "Successfully loaded certificates from fetch."); - } - } else { - log_warn(LD_DIR, - "Couldn't figure out what to do with fetched certificates for " - "unknown resource %s", - conn->requested_resource); + /* launches router downloads as needed */ + routers_update_all_from_networkstatus(now, 3); + update_microdescs_from_networkstatus(now); + update_microdesc_downloads(now); + directory_info_has_arrived(now, 0, 0); + if (authdir_mode_v3(get_options())) { + sr_act_post_consensus( + networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); + } + log_info(LD_DIR, "Successfully loaded consensus."); + + tor_free(new_consensus); + return 0; +} + +/** + * Handler function: processes a response to a request for one or more + * authority certificates + **/ +static int +handle_response_fetch_certificate(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + if (status_code != 200) { + log_warn(LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/keys/%s\".", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); + return -1; + } + log_info(LD_DIR,"Received authority certificates (body size %d) from " + "server '%s:%d'", + (int)body_len, conn->base_.address, conn->base_.port); + + /* + * Tell trusted_dirs_load_certs_from_string() whether it was by fp + * or fp-sk pair. + */ + int src_code = -1; + if (!strcmpstart(conn->requested_resource, "fp/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST; + } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST; + } + + if (src_code != -1) { + if (trusted_dirs_load_certs_from_string(body, src_code, 1, + conn->identity_digest)<0) { + log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); + } else { + time_t now = approx_time(); + directory_info_has_arrived(now, 0, 0); + log_info(LD_DIR, "Successfully loaded certificates from fetch."); } + } else { + log_warn(LD_DIR, + "Couldn't figure out what to do with fetched certificates for " + "unknown resource %s", + conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) { - const char *msg; - int st; - log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - log_warn(LD_DIR, + return 0; +} + +/** + * Handler function: processes a response to a request for an authority's + * current networkstatus vote. + **/ +static int +handle_response_fetch_status_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg; + int st; + log_info(LD_DIR,"Got votes (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, "Received http status code %d (%s) from server " "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".", status_code, escaped(reason), conn->base_.address, conn->base_.port, conn->requested_resource); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - dirvote_add_vote(body, &msg, &st); - if (st > 299) { - log_warn(LD_DIR, "Error adding retrieved vote: %s", msg); - } else { - log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg); - } + return -1; } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { - const char *msg = NULL; - log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - log_warn(LD_DIR, + dirvote_add_vote(body, &msg, &st); + if (st > 299) { + log_warn(LD_DIR, "Error adding retrieved vote: %s", msg); + } else { + log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg); + } + + return 0; +} + +/** + * Handler function: processes a response to a request for the signatures + * that an authority knows about on a given consensus. + **/ +static int +handle_response_fetch_detached_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg = NULL; + log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, "Received http status code %d (%s) from server '%s:%d' while fetching " "\"/tor/status-vote/next/consensus-signatures.z\".", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) { - log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s", - conn->base_.address, conn->base_.port, msg?msg:"???"); - } + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + return -1; + } + if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) { + log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s", + conn->base_.address, conn->base_.port, msg?msg:"???"); } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || - conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { - int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO; - smartlist_t *which = NULL; - int n_asked_for = 0; - int descriptor_digests = conn->requested_resource && - !strcmpstart(conn->requested_resource,"d/"); - log_info(LD_DIR,"Received %s (size %d) from server '%s:%d'", - was_ei ? "extra server info" : "server info", - (int)body_len, conn->base_.address, conn->base_.port); - if (conn->requested_resource && - (!strcmpstart(conn->requested_resource,"d/") || - !strcmpstart(conn->requested_resource,"fp/"))) { - which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource + - (descriptor_digests ? 2 : 3), - which, NULL, 0); - n_asked_for = smartlist_len(which); - } - if (status_code != 200) { - int dir_okay = status_code == 404 || - (status_code == 400 && !strcmp(reason, "Servers unavailable.")); - /* 404 means that it didn't have them; no big deal. - * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */ - log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR, - "Received http status code %d (%s) from server '%s:%d' " - "while fetching \"/tor/server/%s\". I'll try again soon.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource); - if (!which) { - connection_dir_download_routerdesc_failed(conn); - } else { - dir_routerdesc_download_failed(which, status_code, - conn->router_purpose, - was_ei, descriptor_digests); - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - } - tor_free(body); tor_free(headers); tor_free(reason); - return dir_okay ? 0 : -1; - } - /* Learn the routers, assuming we requested by fingerprint or "all" - * or "authority". - * - * We use "authority" to fetch our own descriptor for - * testing, and to fetch bridge descriptors for bootstrapping. Ignore - * the output of "authority" requests unless we are using bridges, - * since otherwise they'll be the response from reachability tests, - * and we don't really want to add that to our routerlist. */ - if (which || (conn->requested_resource && - (!strcmpstart(conn->requested_resource, "all") || - (!strcmpstart(conn->requested_resource, "authority") && - get_options()->UseBridges)))) { - /* as we learn from them, we remove them from 'which' */ - if (was_ei) { - router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which, - descriptor_digests); - } else { - //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, - // descriptor_digests, conn->router_purpose); - if (load_downloaded_routers(body, which, descriptor_digests, - conn->router_purpose, - conn->base_.address)) - directory_info_has_arrived(now, 0, 0); - } - } - if (which) { /* mark remaining ones as failed */ - log_info(LD_DIR, "Received %d/%d %s requested from %s:%d", - n_asked_for-smartlist_len(which), n_asked_for, - was_ei ? "extra-info documents" : "router descriptors", - conn->base_.address, (int)conn->base_.port); - if (smartlist_len(which)) { - dir_routerdesc_download_failed(which, status_code, - conn->router_purpose, - was_ei, descriptor_digests); - } - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - } - if (directory_conn_is_self_reachability_test(conn)) - router_dirport_found_reachable(); - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) { - smartlist_t *which = NULL; - log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " - "size %d) from server '%s:%d'", - status_code, (int)body_len, conn->base_.address, - conn->base_.port); - tor_assert(conn->requested_resource && - !strcmpstart(conn->requested_resource, "d/")); + return 0; +} + +/** + * Handler function: processes a response to a request for a group of server + * descriptors or an extrainfo documents. + **/ +static int +handle_response_fetch_desc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO; + smartlist_t *which = NULL; + int n_asked_for = 0; + int descriptor_digests = conn->requested_resource && + !strcmpstart(conn->requested_resource,"d/"); + log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'", + was_ei ? "extra server info" : "server info", + (int)body_len, conn->base_.address, conn->base_.port); + if (conn->requested_resource && + (!strcmpstart(conn->requested_resource,"d/") || + !strcmpstart(conn->requested_resource,"fp/"))) { which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+2, - which, NULL, - DSR_DIGEST256|DSR_BASE64); - if (status_code != 200) { - log_info(LD_DIR, "Received status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " - "soon.", - status_code, escaped(reason), conn->base_.address, - (int)conn->base_.port, conn->requested_resource); - dir_microdesc_download_failed(which, status_code); + dir_split_resource_into_fingerprints(conn->requested_resource + + (descriptor_digests ? 2 : 3), + which, NULL, 0); + n_asked_for = smartlist_len(which); + } + if (status_code != 200) { + int dir_okay = status_code == 404 || + (status_code == 400 && !strcmp(reason, "Servers unavailable.")); + /* 404 means that it didn't have them; no big deal. + * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */ + log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR, + "Received http status code %d (%s) from server '%s:%d' " + "while fetching \"/tor/server/%s\". I'll try again soon.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + if (!which) { + connection_dir_download_routerdesc_failed(conn); + } else { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); smartlist_free(which); - tor_free(body); tor_free(headers); tor_free(reason); - return 0; + } + return dir_okay ? 0 : -1; + } + /* Learn the routers, assuming we requested by fingerprint or "all" + * or "authority". + * + * We use "authority" to fetch our own descriptor for + * testing, and to fetch bridge descriptors for bootstrapping. Ignore + * the output of "authority" requests unless we are using bridges, + * since otherwise they'll be the response from reachability tests, + * and we don't really want to add that to our routerlist. */ + if (which || (conn->requested_resource && + (!strcmpstart(conn->requested_resource, "all") || + (!strcmpstart(conn->requested_resource, "authority") && + get_options()->UseBridges)))) { + /* as we learn from them, we remove them from 'which' */ + if (was_ei) { + router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which, + descriptor_digests); } else { - smartlist_t *mds; - mds = microdescs_add_to_cache(get_microdesc_cache(), - body, body+body_len, SAVED_NOWHERE, 0, - now, which); - if (smartlist_len(which)) { - /* Mark remaining ones as failed. */ - dir_microdesc_download_failed(which, status_code); - } - if (mds && smartlist_len(mds)) { - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); - directory_info_has_arrived(now, 0, 1); + //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, + // descriptor_digests, conn->router_purpose); + if (load_downloaded_routers(body, which, descriptor_digests, + conn->router_purpose, + conn->base_.address)) { + time_t now = approx_time(); + directory_info_has_arrived(now, 0, 0); } - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - smartlist_free(mds); } } + if (which) { /* mark remaining ones as failed */ + log_info(LD_DIR, "Received %d/%d %s requested from %s:%d", + n_asked_for-smartlist_len(which), n_asked_for, + was_ei ? "extra-info documents" : "router descriptors", + conn->base_.address, (int)conn->base_.port); + if (smartlist_len(which)) { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); + } + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + } + if (directory_conn_is_self_reachability_test(conn)) + router_dirport_found_reachable(); - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR) { - switch (status_code) { - case 200: { - dir_server_t *ds = - router_get_trusteddirserver_by_digest(conn->identity_digest); - char *rejected_hdr = http_get_header(headers, - "X-Descriptor-Not-New: "); - if (rejected_hdr) { - if (!strcmp(rejected_hdr, "Yes")) { - log_info(LD_GENERAL, - "Authority '%s' declined our descriptor (not new)", - ds->nickname); - /* XXXX use this information; be sure to upload next one - * sooner. -NM */ - /* XXXX++ On further thought, the task above implies that we're - * basing our regenerate-descriptor time on when we uploaded the - * last descriptor, not on the published time of the last - * descriptor. If those are different, that's a bad thing to - * do. -NM */ - } - tor_free(rejected_hdr); - } - log_info(LD_GENERAL,"eof (status 200) after uploading server " - "descriptor: finished."); - control_event_server_status( - LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d", - conn->base_.address, conn->base_.port); - - ds->has_accepted_serverdesc = 1; - if (directories_have_accepted_server_descriptor()) - control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR"); - } - break; - case 400: - log_warn(LD_GENERAL,"http status 400 (%s) response from " - "dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - control_event_server_status(LOG_WARN, - "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"", - conn->base_.address, conn->base_.port, escaped(reason)); - break; - default: - log_warn(LD_GENERAL, - "http status %d (%s) reason unexpected while uploading " - "descriptor to server '%s:%d').", + return 0; +} + +/** + * Handler function: processes a response to a request for a group of + * microdescriptors + **/ +static int +handle_response_fetch_microdesc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + smartlist_t *which = NULL; + log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " + "body size %d) from server '%s:%d'", + status_code, (int)body_len, conn->base_.address, + conn->base_.port); + tor_assert(conn->requested_resource && + !strcmpstart(conn->requested_resource, "d/")); + which = smartlist_new(); + dir_split_resource_into_fingerprints(conn->requested_resource+2, + which, NULL, + DSR_DIGEST256|DSR_BASE64); + if (status_code != 200) { + log_info(LD_DIR, "Received status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " + "soon.", status_code, escaped(reason), conn->base_.address, - conn->base_.port); - break; + (int)conn->base_.port, conn->requested_resource); + dir_microdesc_download_failed(which, status_code); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + return 0; + } else { + smartlist_t *mds; + time_t now = approx_time(); + mds = microdescs_add_to_cache(get_microdesc_cache(), + body, body+body_len, SAVED_NOWHERE, 0, + now, which); + if (smartlist_len(which)) { + /* Mark remaining ones as failed. */ + dir_microdesc_download_failed(which, status_code); + } + if (mds && smartlist_len(mds)) { + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); + directory_info_has_arrived(now, 0, 1); } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + smartlist_free(mds); } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE) { - switch (status_code) { - case 200: { - log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d", + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload our + * router descriptor. + **/ +static int +handle_response_upload_dir(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *headers = args->headers; + + switch (status_code) { + case 200: { + dir_server_t *ds = + router_get_trusteddirserver_by_digest(conn->identity_digest); + char *rejected_hdr = http_get_header(headers, + "X-Descriptor-Not-New: "); + if (rejected_hdr) { + if (!strcmp(rejected_hdr, "Yes")) { + log_info(LD_GENERAL, + "Authority '%s' declined our descriptor (not new)", + ds->nickname); + /* XXXX use this information; be sure to upload next one + * sooner. -NM */ + /* XXXX++ On further thought, the task above implies that we're + * basing our regenerate-descriptor time on when we uploaded the + * last descriptor, not on the published time of the last + * descriptor. If those are different, that's a bad thing to + * do. -NM */ + } + tor_free(rejected_hdr); + } + log_info(LD_GENERAL,"eof (status 200) after uploading server " + "descriptor: finished."); + control_event_server_status( + LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d", conn->base_.address, conn->base_.port); - } - break; - case 400: - log_warn(LD_DIR,"http status 400 (%s) response after uploading " - "vote to dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - break; - default: - log_warn(LD_GENERAL, - "http status %d (%s) reason unexpected while uploading " - "vote to server '%s:%d').", + + ds->has_accepted_serverdesc = 1; + if (directories_have_accepted_server_descriptor()) + control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR"); + } + break; + case 400: + log_warn(LD_GENERAL,"http status 400 (%s) response from " + "dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + control_event_server_status(LOG_WARN, + "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"", + conn->base_.address, conn->base_.port, escaped(reason)); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "descriptor to server '%s:%d'. Possibly the server is " + "misconfigured?", status_code, escaped(reason), conn->base_.address, conn->base_.port); - break; - } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ + break; } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES) { - switch (status_code) { - case 200: { - log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d", - conn->base_.address, conn->base_.port); - } - break; - case 400: - log_warn(LD_DIR,"http status 400 (%s) response after uploading " - "signatures to dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - break; - default: - log_warn(LD_GENERAL, - "http status %d (%s) reason unexpected while uploading " - "signatures to server '%s:%d').", + return 0; +} + +/** + * Handler function: processes a response to POST request to upload our + * own networkstatus vote. + **/ +static int +handle_response_upload_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "vote to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "vote to server '%s:%d'.", status_code, escaped(reason), conn->base_.address, conn->base_.port); - break; - } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ - } - - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { - #define SEND_HS_DESC_FAILED_EVENT(reason) ( \ - control_event_hs_descriptor_failed(conn->rend_data, \ - conn->identity_digest, \ - reason) ) - #define SEND_HS_DESC_FAILED_CONTENT() ( \ - control_event_hs_descriptor_content(rend_data_get_address(conn->rend_data), \ - conn->requested_resource, \ + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + return 0; +} + +/** + * Handler function: processes a response to POST request to upload our + * view of the signatures on the current consensus. + **/ +static int +handle_response_upload_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "signatures to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "signatures to server '%s:%d'.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + + return 0; +} + +/** + * Handler function: processes a response to a request for a v2 hidden service + * descriptor. + **/ +static int +handle_response_fetch_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + +#define SEND_HS_DESC_FAILED_EVENT(reason) \ + (control_event_hs_descriptor_failed(conn->rend_data, \ conn->identity_digest, \ - NULL) ) - tor_assert(conn->rend_data); - log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " - "(%s))", - (int)body_len, status_code, escaped(reason)); - switch (status_code) { - case 200: - { - rend_cache_entry_t *entry = NULL; - - if (rend_cache_store_v2_desc_as_client(body, - conn->requested_resource, conn->rend_data, &entry) < 0) { - log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " - "Retrying at another directory."); - /* We'll retry when connection_about_to_close_connection() - * cleans this dir conn up. */ - SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); - SEND_HS_DESC_FAILED_CONTENT(); - } else { - char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; - /* Should never be NULL here if we found the descriptor. */ - tor_assert(entry); - rend_get_service_id(entry->parsed->pk, service_id); - - /* success. notify pending connections about this. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor."); - control_event_hs_descriptor_received(service_id, - conn->rend_data, - conn->identity_digest); - control_event_hs_descriptor_content(service_id, - conn->requested_resource, - conn->identity_digest, - body); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; - rend_client_desc_trynow(service_id); - memwipe(service_id, 0, sizeof(service_id)); - } - break; - } - case 404: - /* Not there. We'll retry when - * connection_about_to_close_connection() cleans this conn up. */ - log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " - "Retrying at another directory."); - SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); - SEND_HS_DESC_FAILED_CONTENT(); - break; - case 400: - log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " - "http status 400 (%s). Dirserver didn't like our " - "v2 rendezvous query? Retrying at another directory.", - escaped(reason)); - SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); - SEND_HS_DESC_FAILED_CONTENT(); - break; - default: - log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " - "http status %d (%s) response unexpected while " - "fetching v2 hidden service descriptor (server '%s:%d'). " - "Retrying at another directory.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + reason)) +#define SEND_HS_DESC_FAILED_CONTENT() \ + (control_event_hs_descriptor_content( \ + rend_data_get_address(conn->rend_data), \ + conn->requested_resource, \ + conn->identity_digest, \ + NULL)) + + tor_assert(conn->rend_data); + log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d " + "(%s))", + (int)body_len, status_code, escaped(reason)); + switch (status_code) { + case 200: + { + rend_cache_entry_t *entry = NULL; + + if (rend_cache_store_v2_desc_as_client(body, + conn->requested_resource, + conn->rend_data, &entry) < 0) { + log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " + "Retrying at another directory."); + /* We'll retry when connection_about_to_close_connection() + * cleans this dir conn up. */ + SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); SEND_HS_DESC_FAILED_CONTENT(); - break; + } else { + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + /* Should never be NULL here if we found the descriptor. */ + tor_assert(entry); + rend_get_service_id(entry->parsed->pk, service_id); + + /* success. notify pending connections about this. */ + log_info(LD_REND, "Successfully fetched v2 rendezvous " + "descriptor."); + control_event_hs_descriptor_received(service_id, + conn->rend_data, + conn->identity_digest); + control_event_hs_descriptor_content(service_id, + conn->requested_resource, + conn->identity_digest, + body); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; + rend_client_desc_trynow(service_id); + memwipe(service_id, 0, sizeof(service_id)); + } + break; } + case 404: + /* Not there. We'll retry when + * connection_about_to_close_connection() cleans this conn up. */ + log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " + "Retrying at another directory."); + SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + case 400: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status 400 (%s). Dirserver didn't like our " + "v2 rendezvous query? Retrying at another directory.", + escaped(reason)); + SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + default: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status %d (%s) response unexpected while " + "fetching v2 hidden service descriptor (server '%s:%d'). " + "Retrying at another directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { - #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \ - control_event_hs_descriptor_upload_failed( \ - conn->identity_digest, \ - rend_data_get_address(conn->rend_data), \ - reason) ) - log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " - "(%s))", - status_code, escaped(reason)); - /* Without the rend data, we'll have a problem identifying what has been - * uploaded for which service. */ - tor_assert(conn->rend_data); - switch (status_code) { - case 200: - log_info(LD_REND, - "Uploading rendezvous descriptor: finished with status " - "200 (%s)", escaped(reason)); - control_event_hs_descriptor_uploaded(conn->identity_digest, - rend_data_get_address(conn->rend_data)); - rend_service_desc_has_uploaded(conn->rend_data); - break; - case 400: - log_warn(LD_REND,"http status 400 (%s) response from dirserver " - "'%s:%d'. Malformed rendezvous descriptor?", - escaped(reason), conn->base_.address, conn->base_.port); - SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); - break; - default: - log_warn(LD_REND,"http status %d (%s) response unexpected (server " - "'%s:%d').", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); - break; - } + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload a v2 + * hidden service descriptor. + **/ +static int +handle_response_upload_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + +#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) \ + (control_event_hs_descriptor_upload_failed( \ + conn->identity_digest, \ + rend_data_get_address(conn->rend_data), \ + reason)) + + log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " + "(%s))", + status_code, escaped(reason)); + /* Without the rend data, we'll have a problem identifying what has been + * uploaded for which service. */ + tor_assert(conn->rend_data); + switch (status_code) { + case 200: + log_info(LD_REND, + "Uploading rendezvous descriptor: finished with status " + "200 (%s)", escaped(reason)); + control_event_hs_descriptor_uploaded(conn->identity_digest, + rend_data_get_address(conn->rend_data)); + rend_service_desc_has_uploaded(conn->rend_data); + break; + case 400: + log_warn(LD_REND,"http status 400 (%s) response from dirserver " + "'%s:%d'. Malformed rendezvous descriptor?", + escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); + break; + default: + log_warn(LD_REND,"http status %d (%s) response unexpected (server " + "'%s:%d').", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); + break; } - tor_free(body); tor_free(headers); tor_free(reason); + return 0; } @@ -2747,14 +3316,114 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, /** As write_http_response_header_impl, but sets encoding and content-typed * based on whether the response will be <b>compressed</b> or not. */ static void -write_http_response_header(dir_connection_t *conn, ssize_t length, - int compressed, long cache_lifetime) +write_http_response_headers(dir_connection_t *conn, ssize_t length, + compress_method_t method, + const char *extra_headers, long cache_lifetime) { + const char *methodname = compression_method_get_name(method); + const char *doctype; + if (method == NO_METHOD) + doctype = "text/plain"; + else + doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - compressed?"application/octet-stream":"text/plain", - compressed?"deflate":"identity", - NULL, - cache_lifetime); + doctype, + methodname, + extra_headers, + cache_lifetime); +} + +/** As write_http_response_headers, but assumes extra_headers is NULL */ +static void +write_http_response_header(dir_connection_t *conn, ssize_t length, + compress_method_t method, + long cache_lifetime) +{ + write_http_response_headers(conn, length, method, NULL, cache_lifetime); +} + +/** Array of compression methods to use (if supported) for serving + * precompressed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_precompressed[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Array of compression methods to use (if supported) for serving + * streamed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_streaming_compression[] = { + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Array of allowed compression methods to use (if supported) when receiving a + * response from a request that was required to be anonymous. */ +static compress_method_t client_meth_allowed_anonymous_compression[] = { + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>, + * and convert them to a bitfield where compression method x is supported if + * and only if 1 << x is set in the bitfield. */ +STATIC unsigned +parse_accept_encoding_header(const char *h) +{ + unsigned result = (1u << NO_METHOD); + smartlist_t *methods = smartlist_new(); + smartlist_split_string(methods, h, ",", + SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + SMARTLIST_FOREACH_BEGIN(methods, const char *, m) { + compress_method_t method = compression_method_get_by_name(m); + if (method != UNKNOWN_METHOD) { + tor_assert(((unsigned)method) < 8*sizeof(unsigned)); + result |= (1u << method); + } + } SMARTLIST_FOREACH_END(m); + SMARTLIST_FOREACH_BEGIN(methods, char *, m) { + tor_free(m); + } SMARTLIST_FOREACH_END(m); + smartlist_free(methods); + return result; +} + +/** Array of compression methods to use (if supported) for requesting + * compressed data, ordered from best to worst. */ +static compress_method_t client_meth_pref[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Return a newly allocated string containing a comma separated list of + * supported encodings. */ +STATIC char * +accept_encoding_header(void) +{ + smartlist_t *methods = smartlist_new(); + char *header = NULL; + compress_method_t method; + unsigned i; + + for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { + method = client_meth_pref[i]; + if (tor_compress_supports_method(method)) + smartlist_add(methods, (char *)compression_method_get_name(method)); + } + + header = smartlist_join_strings(methods, ", ", 0, NULL); + smartlist_free(methods); + + return header; } /** Decide whether a client would accept the consensus we have. @@ -2772,48 +3441,45 @@ write_http_response_header(dir_connection_t *conn, ssize_t length, * consensus, 0 otherwise. */ int -client_likes_consensus(networkstatus_t *v, const char *want_url) +client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url) { - smartlist_t *want_authorities = smartlist_new(); + smartlist_t *voters = smartlist_new(); int need_at_least; int have = 0; + if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) { + return 1; // We don't know the voters; assume the client won't mind. */ + } + + smartlist_t *want_authorities = smartlist_new(); dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { - char want_digest[DIGEST_LEN]; - size_t want_len = strlen(d)/2; - if (want_len > DIGEST_LEN) - want_len = DIGEST_LEN; - - if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) - != (int) want_len) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Failed to decode requested authority digest %s.", escaped(d)); - continue; - }; - SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { - if (smartlist_len(vi->sigs) && - tor_memeq(vi->identity_digest, want_digest, want_len)) { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) { + + SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) { + if (!strcasecmpstart(digest, want_digest)) { have++; break; }; - } SMARTLIST_FOREACH_END(vi); + } SMARTLIST_FOREACH_END(digest); /* early exit, if we already have enough */ if (have >= need_at_least) break; - } SMARTLIST_FOREACH_END(d); + } SMARTLIST_FOREACH_END(want_digest); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); + SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp)); + smartlist_free(voters); return (have >= need_at_least); } /** Return the compression level we should use for sending a compressed * response of size <b>n_bytes</b>. */ -STATIC zlib_compression_level_t +STATIC compression_level_t choose_compression_level(ssize_t n_bytes) { if (! have_been_under_memory_pressure()) { @@ -2831,8 +3497,9 @@ choose_compression_level(ssize_t n_bytes) /** Information passed to handle a GET request. */ typedef struct get_handler_args_t { - /** True if the client asked for compressed data. */ - int compressed; + /** Bitmask of compression methods that the client said (or implied) it + * supported. */ + unsigned compression_supported; /** If nonzero, the time included an if-modified-since header with this * value. */ time_t if_modified_since; @@ -2906,8 +3573,9 @@ directory_handle_command_get,(dir_connection_t *conn, const char *headers, { char *url, *url_mem, *header; time_t if_modified_since = 0; - int compressed; + int zlib_compressed_in_url; size_t url_len; + unsigned compression_methods_supported; /* We ignore the body of a GET request. */ (void)req_body; @@ -2938,17 +3606,31 @@ directory_handle_command_get,(dir_connection_t *conn, const char *headers, url_mem = url; url_len = strlen(url); - compressed = url_len > 2 && !strcmp(url+url_len-2, ".z"); - if (compressed) { + + zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z"); + if (zlib_compressed_in_url) { url[url_len-2] = '\0'; url_len -= 2; } + if ((header = http_get_header(headers, "Accept-Encoding: "))) { + compression_methods_supported = parse_accept_encoding_header(header); + tor_free(header); + } else { + compression_methods_supported = (1u << NO_METHOD); + } + if (zlib_compressed_in_url) { + compression_methods_supported |= (1u << ZLIB_METHOD); + } + + /* Remove all methods that we don't both support. */ + compression_methods_supported &= tor_compress_get_supported_method_bitmask(); + get_handler_args_t args; args.url = url; args.headers = headers; args.if_modified_since = if_modified_since; - args.compressed = compressed; + args.compression_supported = compression_methods_supported; int i, result = -1; for (i = 0; url_table[i].string; ++i) { @@ -2998,20 +3680,25 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) return 0; } -/** Warn that the consensus <b>v</b> of type <b>flavor</b> is too old and will - * not be served to clients. Rate-limit the warning to avoid logging an entry - * on every request. +/** Warn that the cached consensus <b>consensus</b> of type + * <b>flavor</b> is too old and will not be served to clients. Rate-limit the + * warning to avoid logging an entry on every request. */ static void -warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) +warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, + const char *flavor, time_t now) { #define TOO_OLD_WARNING_INTERVAL (60*60) static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); char timestamp[ISO_TIME_LEN+1]; + time_t valid_until; char *dupes; + if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) + return; + if ((dupes = rate_limit_log(&warned, now))) { - format_local_iso_time(timestamp, v->valid_until); + format_local_iso_time(timestamp, valid_until); log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " "serve it to clients. It was valid until %s local time and we " "continued to serve it for up to 24 hours after it expired.%s", @@ -3020,139 +3707,465 @@ warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) } } -/** Helper function for GET /tor/status-vote/current/consensus +/** + * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into + * <b>digest</b>. Return 0 on success. On failure, report that the hash came + * from <b>location</b>, report that we are taking <b>action</b> with it, and + * return -1. */ static int -handle_get_current_consensus(dir_connection_t *conn, - const get_handler_args_t *args) +parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location, + const char *action) { - const char *url = args->url; - const int compressed = args->compressed; - const time_t if_modified_since = args->if_modified_since; + if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) == + DIGEST256_LEN) { + return 0; + } else { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "%s contained bogus digest %s; %s.", + location, escaped(hex), action); + return -1; + } +} - { - /* v3 network status fetch. */ - smartlist_t *dir_fps = smartlist_new(); - long lifetime = NETWORKSTATUS_CACHE_LIFETIME; +/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>, + * set <b>digest_out<b> to a new smartlist containing every 256-bit + * hex-encoded digest listed in that header and return 0. Otherwise return + * -1. */ +static int +parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) +{ + char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER); + if (hdr == NULL) { + return -1; + } + smartlist_t *hex_digests = smartlist_new(); + *digests_out = smartlist_new(); + smartlist_split_string(hex_digests, hdr, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) { + uint8_t digest[DIGEST256_LEN]; + if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header", + "ignoring")) { + smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest))); + } + } SMARTLIST_FOREACH_END(hex); + SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp)); + smartlist_free(hex_digests); + tor_free(hdr); + return 0; +} - networkstatus_t *v; - time_t now = time(NULL); - const char *want_fps = NULL; - char *flavor = NULL; - int flav = FLAV_NS; -#define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" -#define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" - /* figure out the flavor if any, and who we wanted to sign the thing */ - if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { - const char *f, *cp; - f = url + strlen(CONSENSUS_FLAVORED_PREFIX); - cp = strchr(f, '/'); - if (cp) { - want_fps = cp+1; - flavor = tor_strndup(f, cp-f); - } else { - flavor = tor_strdup(f); +/** Fallback compression method. The fallback compression method is used in + * case a client requests a non-compressed document. We only store compressed + * documents, so we use this compression method to fetch the document and let + * the spooling system do the streaming decompression. + */ +#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD + +/** + * Try to find the best consensus diff possible in order to serve a client + * request for a diff from one of the consensuses in <b>digests</b> to the + * current consensus of flavor <b>flav</b>. The client supports the + * compression methods listed in the <b>compression_methods</b> bitfield: + * place the method chosen (if any) into <b>compression_used_out</b>. + */ +static struct consensus_cache_entry_t * +find_best_diff(const smartlist_t *digests, int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + unsigned u; + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + if (0 == (compression_methods & (1u<<method))) + continue; // client doesn't like this one, or we don't have it. + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, + diff_from, DIGEST256_LEN, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; } - flav = networkstatus_parse_flavor_name(flavor); - if (flav < 0) - flav = FLAV_NS; - } else { - if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) - want_fps = url+strlen(CONSENSUS_URL_PREFIX); } + } SMARTLIST_FOREACH_END(diff_from); + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from, + DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + } SMARTLIST_FOREACH_END(diff_from); - v = networkstatus_get_latest_consensus_by_flavor(flav); + return NULL; +} - if (v && !networkstatus_consensus_reasonably_live(v, now)) { - write_http_status_line(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(v, flavor, now); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - tor_free(flavor); - goto done; - } +/** Lookup the cached consensus document by the flavor found in <b>flav</b>. + * The prefered set of compression methods should be listed in the + * <b>compression_methods</b> bitfield. The compression method chosen (if any) + * is stored in <b>compression_used_out</b>. */ +static struct consensus_cache_entry_t * +find_best_consensus(int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + unsigned u; - if (v && want_fps && - !client_likes_consensus(v, want_fps)) { - write_http_status_line(conn, 404, "Consensus not signed by sufficient " - "number of requested authorities"); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); - tor_free(flavor); - goto done; - } + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; - { - char *fp = tor_malloc_zero(DIGEST_LEN); - if (flavor) - strlcpy(fp, flavor, DIGEST_LEN); - tor_free(flavor); - smartlist_add(dir_fps, fp); - } - lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; + if (0 == (compression_methods & (1u<<method))) + continue; - if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ - write_http_status_line(conn, 503, "Network status object unavailable"); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); - goto done; + if (consdiffmgr_find_consensus(&result, flav, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; } + } - if (!dirserv_remove_old_statuses(dir_fps, if_modified_since)) { - write_http_status_line(conn, 404, "Not found"); - SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - goto done; - } else if (!smartlist_len(dir_fps)) { - write_http_status_line(conn, 304, "Not modified"); - SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); - goto done; - } + if (consdiffmgr_find_consensus(&result, flav, + FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } - size_t dlen = dirserv_estimate_data_size(dir_fps, 0, compressed); - if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { - log_debug(LD_DIRSERV, - "Client asked for network status lists, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); - smartlist_free(dir_fps); + return NULL; +} - geoip_note_ns_response(GEOIP_REJECT_BUSY); - goto done; +/** Try to find the best supported compression method possible from a given + * <b>compression_methods</b>. Return NO_METHOD if no mutually supported + * compression method could be found. */ +static compress_method_t +find_best_compression_method(unsigned compression_methods, int stream) +{ + unsigned u; + compress_method_t *methods; + size_t length; + + if (stream) { + methods = srv_meth_pref_streaming_compression; + length = ARRAY_LENGTH(srv_meth_pref_streaming_compression); + } else { + methods = srv_meth_pref_precompressed; + length = ARRAY_LENGTH(srv_meth_pref_precompressed); + } + + for (u = 0; u < length; ++u) { + compress_method_t method = methods[u]; + if (compression_methods & (1u<<method)) + return method; + } + + return NO_METHOD; +} + +/** Check if the given compression method is allowed for a connection that is + * supposed to be anonymous. Returns 1 if the compression method is allowed, + * otherwise 0. */ +STATIC int +allowed_anonymous_connection_compression_method(compress_method_t method) +{ + unsigned u; + + for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression); + ++u) { + compress_method_t allowed_method = + client_meth_allowed_anonymous_compression[u]; + + if (! tor_compress_supports_method(allowed_method)) + continue; + + if (method == allowed_method) + return 1; + } + + return 0; +} + +/** Log a warning when a remote server has sent us a document using a + * compression method that is not allowed for anonymous directory requests. */ +STATIC void +warn_disallowed_anonymous_compression_method(compress_method_t method) +{ + log_fn(LOG_PROTOCOL_WARN, LD_HTTP, + "Received a %s HTTP response, which is not " + "allowed for anonymous directory requests.", + compression_method_get_human_name(method)); +} + +/** Encodes the results of parsing a consensus request to figure out what + * consensus, and possibly what diffs, the user asked for. */ +typedef struct { + /** name of the flavor to retrieve. */ + char *flavor; + /** flavor to retrive, as enum. */ + consensus_flavor_t flav; + /** plus-separated list of authority fingerprints; see + * client_likes_consensus(). Aliases the URL in the request passed to + * parse_consensus_request(). */ + const char *want_fps; + /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses + * to return a diff from. */ + smartlist_t *diff_from_digests; + /** If true, never send a full consensus. If there is no diff, send + * a 404 instead. */ + int diff_only; +} parsed_consensus_request_t; + +/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since + * it is stack-allocated. */ +static void +parsed_consensus_request_clear(parsed_consensus_request_t *req) +{ + if (!req) + return; + tor_free(req->flavor); + if (req->diff_from_digests) { + SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d)); + smartlist_free(req->diff_from_digests); + } + memset(req, 0, sizeof(parsed_consensus_request_t)); +} + +/** + * Parse the URL and relevant headers of <b>args</b> for a current-consensus + * request to learn what flavor of consensus we want, what keys it must be + * signed with, and what diffs we would accept (or demand) instead. Return 0 + * on success and -1 on failure. + */ +static int +parse_consensus_request(parsed_consensus_request_t *out, + const get_handler_args_t *args) +{ + const char *url = args->url; + memset(out, 0, sizeof(parsed_consensus_request_t)); + out->flav = FLAV_NS; + + const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/"; + const char CONSENSUS_FLAVORED_PREFIX[] = + "/tor/status-vote/current/consensus-"; + + /* figure out the flavor if any, and who we wanted to sign the thing */ + const char *after_flavor = NULL; + + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + after_flavor = cp+1; + out->flavor = tor_strndup(f, cp-f); + } else { + out->flavor = tor_strdup(f); + } + int flav = networkstatus_parse_flavor_name(out->flavor); + if (flav < 0) + flav = FLAV_NS; + out->flav = flav; + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + after_flavor = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* see whether we've been asked explicitly for a diff from an older + * consensus. (The user might also have said that a diff would be okay, + * via X-Or-Diff-From-Consensus */ + const char DIFF_COMPONENT[] = "diff/"; + char *diff_hash_in_url = NULL; + if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) { + after_flavor += strlen(DIFF_COMPONENT); + const char *cp = strchr(after_flavor, '/'); + if (cp) { + diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor); + out->want_fps = cp+1; + } else { + diff_hash_in_url = tor_strdup(after_flavor); + out->want_fps = NULL; } + } else { + out->want_fps = after_flavor; + } + + if (diff_hash_in_url) { + uint8_t diff_from[DIGEST256_LEN]; + out->diff_from_digests = smartlist_new(); + out->diff_only = 1; + int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL", + "rejecting"); + tor_free(diff_hash_in_url); + if (ok) { + smartlist_add(out->diff_from_digests, + tor_memdup(diff_from, DIGEST256_LEN)); + } else { + return -1; + } + } else { + parse_or_diff_from_header(&out->diff_from_digests, args->headers); + } - tor_addr_t addr; - if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, - &addr, NULL, - time(NULL)); - geoip_note_ns_response(GEOIP_SUCCESS); - /* Note that a request for a network status has started, so that we - * can measure the download time later on. */ - if (conn->dirreq_id) - geoip_start_dirreq(conn->dirreq_id, dlen, DIRREQ_TUNNELED); - else - geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, - DIRREQ_DIRECT); + return 0; +} + +/** Helper function for GET /tor/status-vote/current/consensus + */ +static int +handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 0); + const time_t if_modified_since = args->if_modified_since; + int clear_spool = 0; + + /* v3 network status fetch. */ + long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + + time_t now = time(NULL); + parsed_consensus_request_t req; + + if (parse_consensus_request(&req, args) < 0) { + write_http_status_line(conn, 404, "Couldn't parse request"); + goto done; + } + + struct consensus_cache_entry_t *cached_consensus = NULL; + + compress_method_t compression_used = NO_METHOD; + if (req.diff_from_digests) { + cached_consensus = find_best_diff(req.diff_from_digests, req.flav, + args->compression_supported, + &compression_used); + } + + if (req.diff_only && !cached_consensus) { + write_http_status_line(conn, 404, "No such diff available"); + // XXXX warn_consensus_is_too_old(v, req.flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } + + if (! cached_consensus) { + cached_consensus = find_best_consensus(req.flav, + args->compression_supported, + &compression_used); + } + + time_t fresh_until, valid_until; + int have_fresh_until = 0, have_valid_until = 0; + if (cached_consensus) { + have_fresh_until = + !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); + have_valid_until = + !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); + } + + if (cached_consensus && have_valid_until && + !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { + write_http_status_line(conn, 404, "Consensus is too old"); + warn_consensus_is_too_old(cached_consensus, req.flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } + + if (cached_consensus && req.want_fps && + !client_likes_consensus(cached_consensus, req.want_fps)) { + write_http_status_line(conn, 404, "Consensus not signed by sufficient " + "number of requested authorities"); + geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); + goto done; + } + + conn->spool = smartlist_new(); + clear_spool = 1; + { + spooled_resource_t *spooled; + if (cached_consensus) { + spooled = spooled_resource_new_from_cache_entry(cached_consensus); + smartlist_add(conn->spool, spooled); } + } - write_http_response_header(conn, -1, compressed, - smartlist_len(dir_fps) == 1 ? lifetime : 0); - conn->fingerprint_stack = dir_fps; - if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; + + size_t size_guess = 0; + int n_expired = 0; + dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, + compress_method != NO_METHOD, + &size_guess, + &n_expired); + + if (!smartlist_len(conn->spool) && !n_expired) { + write_http_status_line(conn, 404, "Not found"); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } else if (!smartlist_len(conn->spool)) { + write_http_status_line(conn, 304, "Not modified"); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + goto done; + } - /* Prime the connection with some data. */ - conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS; - connection_dirserv_flushed_some(conn); + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { + log_debug(LD_DIRSERV, + "Client asked for network status lists, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + geoip_note_ns_response(GEOIP_REJECT_BUSY); goto done; } + tor_addr_t addr; + if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, + &addr, NULL, + time(NULL)); + geoip_note_ns_response(GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (conn->dirreq_id) + geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess, + DIRREQ_DIRECT); + } + + /* Use this header to tell caches that the response depends on the + * X-Or-Diff-From-Consensus header (or lack thereof). */ + const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n"; + + clear_spool = 0; + + // The compress_method might have been NO_METHOD, but we store the data + // compressed. Decompress them using `compression_used`. See fallback code in + // find_best_consensus() and find_best_diff(). + write_http_response_headers(conn, -1, + compress_method == NO_METHOD ? + NO_METHOD : compression_used, + vary_header, + smartlist_len(conn->spool) == 1 ? lifetime : 0); + + if (compress_method == NO_METHOD && smartlist_len(conn->spool)) + conn->compress_state = tor_compress_new(0, compression_used, + HIGH_COMPRESSION); + + /* Prime the connection with some data. */ + const int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); + goto done; + done: + parsed_consensus_request_clear(&req); + if (clear_spool) { + dir_conn_clear_spool(conn); + } return 0; } @@ -3162,7 +4175,8 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); { int current; ssize_t body_len = 0; @@ -3219,11 +4233,12 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compressed ? d->dir_z_len : d->dir_len); + body_len += compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); - if (compressed) { + if (compress_method != NO_METHOD) { estimated_len += ln/2; } else { body_len += ln; estimated_len += ln; @@ -3234,24 +4249,27 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) write_http_status_line(conn, 503, "Directory busy, try again later"); goto vote_done; } - write_http_response_header(conn, body_len ? body_len : -1, compressed, + write_http_response_header(conn, body_len ? body_len : -1, + compress_method, lifetime); if (smartlist_len(items)) { - if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(estimated_len)); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf_zlib(c, strlen(c), conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress(c, strlen(c), conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf(c, strlen(c), TO_CONN(conn))); } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compressed ? d->dir_z : d->dir, - compressed ? d->dir_z_len : d->dir_len, + connection_write_to_buf(compress_method != NO_METHOD ? + d->dir_compressed : d->dir, + compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } vote_done: @@ -3269,44 +4287,51 @@ static int handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + int clear_spool = 1; { - smartlist_t *fps = smartlist_new(); + conn->spool = smartlist_new(); - dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), - fps, NULL, + dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"), + DIR_SPOOL_MICRODESC, + conn->spool, NULL, DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); - if (!dirserv_have_any_microdesc(fps)) { + size_t size_guess = 0; + dirserv_spool_remove_missing_and_guess_size(conn, 0, + compress_method != NO_METHOD, + &size_guess, NULL); + if (smartlist_len(conn->spool) == 0) { write_http_status_line(conn, 404, "Not found"); - SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); - smartlist_free(fps); goto done; } - size_t dlen = dirserv_estimate_microdesc_size(fps, compressed); - if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { log_info(LD_DIRSERV, "Client asked for server descriptors, but we've been " "writing too many bytes lately. Sending 503 Dir busy."); write_http_status_line(conn, 503, "Directory busy, try again later"); - SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); - smartlist_free(fps); goto done; } - write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); - conn->dir_spool_src = DIR_SPOOL_MICRODESC; - conn->fingerprint_stack = fps; + clear_spool = 0; + write_http_response_header(conn, -1, + compress_method, + MICRODESC_CACHE_LIFETIME); - if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(dlen)); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(size_guess)); - connection_dirserv_flushed_some(conn); + const int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); goto done; } done: + if (clear_spool) { + dir_conn_clear_spool(conn); + } return 0; } @@ -3316,71 +4341,92 @@ static int handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); const or_options_t *options = get_options(); + int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { - size_t dlen; int res; - const char *msg; + const char *msg = NULL; int cache_lifetime = 0; int is_extra = !strcmpstart(url,"/tor/extra/"); url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/"); - conn->fingerprint_stack = smartlist_new(); - res = dirserv_get_routerdesc_fingerprints(conn->fingerprint_stack, url, - &msg, - !connection_dir_is_encrypted(conn), - is_extra); - - if (!strcmpstart(url, "fp/")) { - if (smartlist_len(conn->fingerprint_stack) == 1) - cache_lifetime = ROUTERDESC_CACHE_LIFETIME; - } else if (!strcmpstart(url, "authority")) { - cache_lifetime = ROUTERDESC_CACHE_LIFETIME; - } else if (!strcmpstart(url, "all")) { - cache_lifetime = FULL_DIR_CACHE_LIFETIME; - } else if (!strcmpstart(url, "d/")) { - if (smartlist_len(conn->fingerprint_stack) == 1) - cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; - } - if (!strcmpstart(url, "d/")) - conn->dir_spool_src = + dir_spool_source_t source; + time_t publish_cutoff = 0; + if (!strcmpstart(url, "d/")) { + source = is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST; - else - conn->dir_spool_src = + } else { + source = is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP; + /* We only want to apply a publish cutoff when we're requesting + * resources by fingerprint. */ + publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; + } + + conn->spool = smartlist_new(); + res = dirserv_get_routerdesc_spool(conn->spool, url, + source, + connection_dir_is_encrypted(conn), + &msg); + + if (!strcmpstart(url, "all")) { + cache_lifetime = FULL_DIR_CACHE_LIFETIME; + } else if (smartlist_len(conn->spool) == 1) { + cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; + } - if (!dirserv_have_any_serverdesc(conn->fingerprint_stack, - conn->dir_spool_src)) { - res = -1; - msg = "Not found"; + size_t size_guess = 0; + int n_expired = 0; + dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff, + compress_method != NO_METHOD, + &size_guess, &n_expired); + + /* If we are the bridge authority and the descriptor is a bridge + * descriptor, remember that we served this descriptor for desc stats. */ + /* XXXX it's a bit of a kludge to have this here. */ + if (get_options()->BridgeAuthoritativeDir && + source == DIR_SPOOL_SERVER_BY_FP) { + SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) { + const routerinfo_t *router = + router_get_by_id_digest((const char *)spooled->digest); + /* router can be NULL here when the bridge auth is asked for its own + * descriptor. */ + if (router && router->purpose == ROUTER_PURPOSE_BRIDGE) + rep_hist_note_desc_served(router->cache_info.identity_digest); + } SMARTLIST_FOREACH_END(spooled); } - if (res < 0) + if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) { + if (msg == NULL) + msg = "Not found"; write_http_status_line(conn, 404, msg); - else { - dlen = dirserv_estimate_data_size(conn->fingerprint_stack, - 1, compressed); - if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + } else { + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { log_info(LD_DIRSERV, "Client asked for server descriptors, but we've been " "writing too many bytes lately. Sending 503 Dir busy."); write_http_status_line(conn, 503, "Directory busy, try again later"); - conn->dir_spool_src = DIR_SPOOL_NONE; + dir_conn_clear_spool(conn); goto done; } - write_http_response_header(conn, -1, compressed, cache_lifetime); - if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(dlen)); + write_http_response_header(conn, -1, compress_method, cache_lifetime); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(size_guess)); + clear_spool = 0; /* Prime the connection with some data. */ - connection_dirserv_flushed_some(conn); + int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); } goto done; } done: - return 0; + if (clear_spool) + dir_conn_clear_spool(conn); + return 0; } /** Helper function for GET /tor/keys/... @@ -3389,7 +4435,8 @@ static int handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new(); @@ -3452,20 +4499,26 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) SMARTLIST_FOREACH(certs, authority_cert_t *, c, len += c->cache_info.signed_descriptor_len); - if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) { + if (global_write_bucket_low(TO_CONN(conn), + compress_method != NO_METHOD ? len/2 : len, + 2)) { write_http_status_line(conn, 503, "Directory busy, try again later"); goto keys_done; } - write_http_response_header(conn, compressed?-1:len, compressed, 60*60); - if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(len)); + write_http_response_header(conn, + compress_method != NO_METHOD ? -1 : len, + compress_method, + 60*60); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress( + c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf(c->cache_info.signed_descriptor_body, @@ -3496,7 +4549,7 @@ handle_get_hs_descriptor_v2(dir_connection_t *conn, safe_str(escaped(query))); switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { case 1: /* valid */ - write_http_response_header(conn, strlen(descp), 0, 0); + write_http_response_header(conn, strlen(descp), NO_METHOD, 0); connection_write_to_buf(descp, strlen(descp), TO_CONN(conn)); break; case 0: /* well-formed but not present */ @@ -3548,7 +4601,7 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn, } /* Found requested descriptor! Pass it to this nice client. */ - write_http_response_header(conn, strlen(desc_str), 0, 0); + write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0); connection_write_to_buf(desc_str, strlen(desc_str), TO_CONN(conn)); done: @@ -3587,7 +4640,7 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, /* all happy now. send an answer. */ status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); size_t dlen = strlen(status); - write_http_response_header(conn, dlen, 0, 0); + write_http_response_header(conn, dlen, NO_METHOD, 0); connection_write_to_buf(status, dlen, TO_CONN(conn)); tor_free(status); goto done; @@ -3604,7 +4657,7 @@ handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) { const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; size_t len = strlen(robots); - write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME); + write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME); connection_write_to_buf(robots, len, TO_CONN(conn)); } return 0; @@ -3727,7 +4780,7 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, if (connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/publish")) { if (rend_cache_store_v2_desc_as_dir(body) < 0) { - log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", + log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.", (int)body_len, conn->base_.address); write_http_status_line(conn, 400, "Invalid v2 service descriptor rejected"); @@ -3770,14 +4823,7 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, conn->base_.address, &msg); tor_assert(msg); - if (r == ROUTER_ADDED_NOTIFY_GENERATOR) { - /* Accepted with a message. */ - log_info(LD_DIRSERV, - "Problematic router descriptor or extra-info from %s " - "(\"%s\").", - conn->base_.address, msg); - write_http_status_line(conn, 400, msg); - } else if (r == ROUTER_ADDED_SUCCESSFULLY) { + if (r == ROUTER_ADDED_SUCCESSFULLY) { write_http_status_line(conn, 200, msg); } else if (WRA_WAS_OUTDATED(r)) { write_http_response_header_impl(conn, -1, NULL, NULL, @@ -3904,7 +4950,7 @@ connection_dir_finished_flushing(dir_connection_t *conn) conn->base_.state = DIR_CONN_STATE_CLIENT_READING; return 0; case DIR_CONN_STATE_SERVER_WRITING: - if (conn->dir_spool_src != DIR_SPOOL_NONE) { + if (conn->spool) { log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!"); connection_mark_for_close(TO_CONN(conn)); } else { @@ -4587,3 +5633,34 @@ dir_split_resource_into_fingerprints(const char *resource, return 0; } +/** As dir_split_resource_into_fingerprints, but instead fills + * <b>spool_out</b> with a list of spoolable_resource_t for the resource + * identified through <b>source</b>. */ +int +dir_split_resource_into_spoolable(const char *resource, + dir_spool_source_t source, + smartlist_t *spool_out, + int *compressed_out, + int flags) +{ + smartlist_t *fingerprints = smartlist_new(); + + tor_assert(flags & (DSR_HEX|DSR_BASE64)); + const size_t digest_len = + (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN; + + int r = dir_split_resource_into_fingerprints(resource, fingerprints, + compressed_out, flags); + /* This is not a very efficient implementation XXXX */ + SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) { + spooled_resource_t *spooled = + spooled_resource_new(source, digest, digest_len); + if (spooled) + smartlist_add(spool_out, spooled); + tor_free(digest); + } SMARTLIST_FOREACH_END(digest); + + smartlist_free(fingerprints); + return r; +} + diff --git a/src/or/directory.h b/src/or/directory.h index 8be1f8e76e..14d5ae9ef4 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -41,45 +41,53 @@ typedef enum { int directory_must_use_begindir(const or_options_t *options); -MOCK_DECL(void, directory_initiate_command_routerstatus, - (const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - struct circuit_guard_state_t *guard_state)); - -void directory_initiate_command_routerstatus_rend(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - struct circuit_guard_state_t *guard_state); +/** + * A directory_request_t describes the information about a directory request + * at the client side. It describes what we're going to ask for, which + * directory we're going to ask for it, how we're going to contact that + * directory, and (in some cases) what to do with it when we're done. + */ +typedef struct directory_request_t directory_request_t; +directory_request_t *directory_request_new(uint8_t dir_purpose); +void directory_request_free(directory_request_t *req); +void directory_request_set_or_addr_port(directory_request_t *req, + const tor_addr_port_t *p); +void directory_request_set_dir_addr_port(directory_request_t *req, + const tor_addr_port_t *p); +void directory_request_set_directory_id_digest(directory_request_t *req, + const char *digest); +void directory_request_set_guard_state(directory_request_t *req, + struct circuit_guard_state_t *state); +void directory_request_set_router_purpose(directory_request_t *req, + uint8_t router_purpose); +void directory_request_set_indirection(directory_request_t *req, + dir_indirection_t indirection); +void directory_request_set_resource(directory_request_t *req, + const char *resource); +void directory_request_set_payload(directory_request_t *req, + const char *payload, + size_t payload_len); +void directory_request_set_if_modified_since(directory_request_t *req, + time_t if_modified_since); +void directory_request_set_rend_query(directory_request_t *req, + const rend_data_t *query); + +void directory_request_set_routerstatus(directory_request_t *req, + const routerstatus_t *rs); +void directory_request_add_header(directory_request_t *req, + const char *key, + const char *val); +MOCK_DECL(void, directory_initiate_request, (directory_request_t *request)); int parse_http_response(const char *headers, int *code, time_t *date, compress_method_t *compression, char **response); -int connection_dir_is_encrypted(dir_connection_t *conn); +int connection_dir_is_encrypted(const dir_connection_t *conn); int connection_dir_reached_eof(dir_connection_t *conn); int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); int connection_dir_finished_connecting(dir_connection_t *conn); void connection_dir_about_to_close(dir_connection_t *dir_conn); -void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, - const tor_addr_t *dir_addr, uint16_t dir_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); #define DSR_HEX (1<<0) #define DSR_BASE64 (1<<1) @@ -88,7 +96,12 @@ void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, int flags); - +enum dir_spool_source_t; +int dir_split_resource_into_spoolable(const char *resource, + enum dir_spool_source_t source, + smartlist_t *spool_out, + int *compressed_out, + int flags); int dir_split_resource_into_fingerprint_pairs(const char *res, smartlist_t *pairs_out); char *directory_dump_request_log(void); @@ -149,6 +162,9 @@ struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, const struct get_handler_args_t *args); STATIC int directory_handle_command(dir_connection_t *conn); +STATIC char *accept_encoding_header(void); +STATIC int allowed_anonymous_connection_compression_method(compress_method_t); +STATIC void warn_disallowed_anonymous_compression_method(compress_method_t); #endif @@ -176,7 +192,7 @@ STATIC int handle_post_hs_descriptor(const char *url, const char *body); STATIC char* authdir_type_to_string(dirinfo_type_t auth); STATIC const char * dir_conn_purpose_to_string(int purpose); STATIC int should_use_directory_guards(const or_options_t *options); -STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC compression_level_t choose_compression_level(ssize_t n_bytes); STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, const or_options_t *options); STATIC void find_dl_min_and_max_delay(download_status_t *dls, @@ -187,6 +203,7 @@ STATIC int next_random_exponential_delay(int delay, int max_delay); STATIC int parse_hs_version_from_post(const char *url, const char *prefix, const char **end_pos); +STATIC unsigned parse_accept_encoding_header(const char *h); #endif #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 250170cbef..408f58b22b 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@ -13,6 +13,7 @@ #include "command.h" #include "connection.h" #include "connection_or.h" +#include "conscache.h" #include "control.h" #include "directory.h" #include "dirserv.h" @@ -81,14 +82,23 @@ dirserv_get_status_impl(const char *fp, const char *nickname, int severity); static void clear_cached_dir(cached_dir_t *d); static const signed_descriptor_t *get_signed_descriptor_by_fp( - const char *fp, - int extrainfo, - time_t publish_cutoff); + const uint8_t *fp, + int extrainfo); static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); static uint32_t dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri); static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri); +static int spooled_resource_lookup_body(const spooled_resource_t *spooled, + int conn_is_encrypted, + const uint8_t **body_out, + size_t *size_out, + time_t *published_out); +static cached_dir_t *spooled_resource_lookup_cached_dir( + const spooled_resource_t *spooled, + time_t *published_out); +static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp); + /************** Fingerprint handling code ************/ /* 1 Historically used to indicate Named */ @@ -274,6 +284,13 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg, return FP_REJECT; } + /* Check for the more usual versions to reject a router first. */ + const uint32_t r = dirserv_get_status_impl(d, router->nickname, + router->addr, router->or_port, + router->platform, msg, severity); + if (r) + return r; + /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc, * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha. * But just in case a relay doesn't provide or lies about its version, or @@ -324,9 +341,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg, } } - return dirserv_get_status_impl(d, router->nickname, - router->addr, router->or_port, - router->platform, msg, severity); + return 0; } /** Return true if there is no point in downloading the router described by @@ -572,6 +587,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, !general ? router_purpose_to_string(purpose) : "", !general ? "\n" : "")<0) { *msg = "Couldn't format annotations"; + /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is + * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */ return -1; } @@ -830,6 +847,9 @@ directory_remove_invalid(void) * Allocate and return a description of the status of the server <b>desc</b>, * for use in a v1-style router-status line. The server is listed * as running iff <b>is_live</b> is true. + * + * This is deprecated: it's only used for controllers that want outputs in + * the old format. */ static char * list_single_server_status(const routerinfo_t *desc, int is_live) @@ -942,6 +962,9 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. + * + * This is deprecated: it's only used for controllers that want outputs in + * the old format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -1177,8 +1200,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len), + d->dir, d->dir_len, ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; @@ -1189,7 +1212,7 @@ static void clear_cached_dir(cached_dir_t *d) { tor_free(d->dir); - tor_free(d->dir_z); + tor_free(d->dir_compressed); memset(d, 0, sizeof(cached_dir_t)); } @@ -1212,6 +1235,7 @@ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, const char *flavor_name, const common_digests_t *digests, + const uint8_t *sha3_as_signed, time_t published) { cached_dir_t *new_networkstatus; @@ -1221,6 +1245,8 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); + memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, + DIGEST256_LEN); old_networkstatus = strmap_set(cached_consensuses, flavor_name, new_networkstatus); if (old_networkstatus) @@ -3053,58 +3079,61 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, * requests, adds identity digests. */ int -dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, - const char **msg, int for_unencrypted_conn, - int is_extrainfo) +dirserv_get_routerdesc_spool(smartlist_t *spool_out, + const char *key, + dir_spool_source_t source, + int conn_is_encrypted, + const char **msg_out) { - int by_id = 1; - *msg = NULL; + *msg_out = NULL; if (!strcmp(key, "all")) { - routerlist_t *rl = router_get_routerlist(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, - smartlist_add(fps_out, - tor_memdup(r->cache_info.identity_digest, DIGEST_LEN))); - /* Treat "all" requests as if they were unencrypted */ - for_unencrypted_conn = 1; + const routerlist_t *rl = router_get_routerlist(); + SMARTLIST_FOREACH_BEGIN(rl->routers, const routerinfo_t *, r) { + spooled_resource_t *spooled; + spooled = spooled_resource_new(source, + (const uint8_t *)r->cache_info.identity_digest, + DIGEST_LEN); + /* Treat "all" requests as if they were unencrypted */ + conn_is_encrypted = 0; + smartlist_add(spool_out, spooled); + } SMARTLIST_FOREACH_END(r); } else if (!strcmp(key, "authority")) { const routerinfo_t *ri = router_get_my_routerinfo(); if (ri) - smartlist_add(fps_out, - tor_memdup(ri->cache_info.identity_digest, DIGEST_LEN)); + smartlist_add(spool_out, + spooled_resource_new(source, + (const uint8_t *)ri->cache_info.identity_digest, + DIGEST_LEN)); } else if (!strcmpstart(key, "d/")) { - by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, - DSR_HEX|DSR_SORT_UNIQ); + dir_split_resource_into_spoolable(key, source, spool_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, - DSR_HEX|DSR_SORT_UNIQ); + dir_split_resource_into_spoolable(key, source, spool_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { - *msg = "Key not recognized"; + *msg_out = "Not found"; return -1; } - if (for_unencrypted_conn) { + if (! conn_is_encrypted) { /* Remove anything that insists it not be sent unencrypted. */ - SMARTLIST_FOREACH_BEGIN(fps_out, char *, cp) { - const signed_descriptor_t *sd; - if (by_id) - sd = get_signed_descriptor_by_fp(cp,is_extrainfo,0); - else if (is_extrainfo) - sd = extrainfo_get_by_descriptor_digest(cp); - else - sd = router_get_by_descriptor_digest(cp); - if (sd && !sd->send_unencrypted) { - tor_free(cp); - SMARTLIST_DEL_CURRENT(fps_out, cp); - } - } SMARTLIST_FOREACH_END(cp); + SMARTLIST_FOREACH_BEGIN(spool_out, spooled_resource_t *, spooled) { + const uint8_t *body = NULL; + size_t bodylen = 0; + int r = spooled_resource_lookup_body(spooled, conn_is_encrypted, + &body, &bodylen, NULL); + if (r < 0 || body == NULL || bodylen == 0) { + SMARTLIST_DEL_CURRENT(spool_out, spooled); + spooled_resource_free(spooled); + } + } SMARTLIST_FOREACH_END(spooled); } - if (!smartlist_len(fps_out)) { - *msg = "Servers unavailable"; + if (!smartlist_len(spool_out)) { + *msg_out = "Servers unavailable"; return -1; } return 0; @@ -3368,416 +3397,497 @@ dirserv_test_reachability(time_t now) ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */ } -/** Given a fingerprint <b>fp</b> which is either set if we're looking for a - * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded - * flavor name if we want a flavored v3 status, return a pointer to the - * appropriate cached dir object, or NULL if there isn't one available. */ -static cached_dir_t * -lookup_cached_dir_by_fp(const char *fp) +/* ========== + * Spooling code. + * ========== */ + +spooled_resource_t * +spooled_resource_new(dir_spool_source_t source, + const uint8_t *digest, size_t digestlen) { - cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_consensuses) { - d = strmap_get(cached_consensuses, "ns"); - } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && - (d = strmap_get(cached_consensuses, fp))) { - /* this here interface is a nasty hack XXXX */; + spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t)); + spooled->spool_source = source; + switch (source) { + case DIR_SPOOL_NETWORKSTATUS: + spooled->spool_eagerly = 0; + break; + case DIR_SPOOL_SERVER_BY_DIGEST: + case DIR_SPOOL_SERVER_BY_FP: + case DIR_SPOOL_EXTRA_BY_DIGEST: + case DIR_SPOOL_EXTRA_BY_FP: + case DIR_SPOOL_MICRODESC: + default: + spooled->spool_eagerly = 1; + break; + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: + tor_assert_unreached(); + break; } - return d; + tor_assert(digestlen <= sizeof(spooled->digest)); + if (digest) + memcpy(spooled->digest, digest, digestlen); + return spooled; } -/** Remove from <b>fps</b> every networkstatus key where both - * a) we have a networkstatus document and - * b) it is not newer than <b>cutoff</b>. +/** + * Create a new spooled_resource_t to spool the contents of <b>entry</b> to + * the user. Return the spooled object on success, or NULL on failure (which + * is probably caused by a failure to map the body of the item from disk). * - * Return 1 if any items were present at all; else return 0. + * Adds a reference to entry's reference counter. */ -int -dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff) -{ - int found_any = 0; - SMARTLIST_FOREACH_BEGIN(fps, char *, digest) { - cached_dir_t *d = lookup_cached_dir_by_fp(digest); - if (!d) - continue; - found_any = 1; - if (d->published <= cutoff) { - tor_free(digest); - SMARTLIST_DEL_CURRENT(fps, digest); - } - } SMARTLIST_FOREACH_END(digest); - - return found_any; +spooled_resource_t * +spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry) +{ + spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t)); + spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY; + spooled->spool_eagerly = 0; + consensus_cache_entry_incref(entry); + spooled->consensus_cache_entry = entry; + + int r = consensus_cache_entry_get_body(entry, + &spooled->cce_body, + &spooled->cce_len); + if (r == 0) { + return spooled; + } else { + spooled_resource_free(spooled); + return NULL; + } } -/** Return the cache-info for identity fingerprint <b>fp</b>, or - * its extra-info document if <b>extrainfo</b> is true. Return - * NULL if not found or if the descriptor is older than - * <b>publish_cutoff</b>. */ -static const signed_descriptor_t * -get_signed_descriptor_by_fp(const char *fp, int extrainfo, - time_t publish_cutoff) +/** Release all storage held by <b>spooled</b>. */ +void +spooled_resource_free(spooled_resource_t *spooled) { - if (router_digest_is_me(fp)) { - if (extrainfo) - return &(router_get_my_extrainfo()->cache_info); - else - return &(router_get_my_routerinfo()->cache_info); - } else { - const routerinfo_t *ri = router_get_by_id_digest(fp); - if (ri && - ri->cache_info.published_on > publish_cutoff) { - if (extrainfo) - return extrainfo_get_by_descriptor_digest( - ri->cache_info.extra_info_digest); - else - return &ri->cache_info; - } + if (spooled == NULL) + return; + + if (spooled->cached_dir_ref) { + cached_dir_decref(spooled->cached_dir_ref); } - return NULL; -} -/** Return true iff we have any of the documents (extrainfo or routerdesc) - * specified by the fingerprints in <b>fps</b> and <b>spool_src</b>. Used to - * decide whether to send a 404. */ -int -dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) -{ - time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH; - SMARTLIST_FOREACH_BEGIN(fps, const char *, fp) { - switch (spool_src) - { - case DIR_SPOOL_EXTRA_BY_DIGEST: - if (extrainfo_get_by_descriptor_digest(fp)) return 1; - break; - case DIR_SPOOL_SERVER_BY_DIGEST: - if (router_get_by_descriptor_digest(fp)) return 1; - break; - case DIR_SPOOL_EXTRA_BY_FP: - case DIR_SPOOL_SERVER_BY_FP: - if (get_signed_descriptor_by_fp(fp, - spool_src == DIR_SPOOL_EXTRA_BY_FP, publish_cutoff)) - return 1; - break; - } - } SMARTLIST_FOREACH_END(fp); - return 0; + if (spooled->consensus_cache_entry) { + consensus_cache_entry_decref(spooled->consensus_cache_entry); + } + + tor_free(spooled); } -/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of - * a microdescriptor we have. */ -int -dirserv_have_any_microdesc(const smartlist_t *fps) +/** When spooling data from a cached_dir_t object, we always add + * at least this much. */ +#define DIRSERV_CACHED_DIR_CHUNK_SIZE 8192 + +/** Return an compression ratio for compressing objects from <b>source</b>. + */ +static double +estimate_compression_ratio(dir_spool_source_t source) { - microdesc_cache_t *cache = get_microdesc_cache(); - SMARTLIST_FOREACH(fps, const char *, fp, - if (microdesc_cache_lookup_by_digest256(cache, fp)) - return 1); - return 0; + /* We should put in better estimates here, depending on the number of + objects and their type */ + (void) source; + return 0.5; } -/** Return an approximate estimate of the number of bytes that will - * be needed to transmit the server descriptors (if is_serverdescs -- - * they can be either d/ or fp/ queries) or networkstatus objects (if - * !is_serverdescs) listed in <b>fps</b>. If <b>compressed</b> is set, - * we guess how large the data will be after compression. +/** Return an estimated number of bytes needed for transmitting the + * resource in <b>spooled</b> on <b>conn</b> * - * The return value is an estimate; it might be larger or smaller. - **/ -size_t -dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, - int compressed) -{ - size_t result; - tor_assert(fps); - if (is_serverdescs) { - int n = smartlist_len(fps); - const routerinfo_t *me = router_get_my_routerinfo(); - result = (me?me->cache_info.signed_descriptor_len:2048) * n; - if (compressed) - result /= 2; /* observed compressibility is between 35 and 55%. */ + * As a convenient side-effect, set *<b>published_out</b> to the resource's + * publication time. + */ +static size_t +spooled_resource_estimate_size(const spooled_resource_t *spooled, + dir_connection_t *conn, + int compressed, + time_t *published_out) +{ + if (spooled->spool_eagerly) { + const uint8_t *body = NULL; + size_t bodylen = 0; + int r = spooled_resource_lookup_body(spooled, + connection_dir_is_encrypted(conn), + &body, &bodylen, + published_out); + if (r == -1 || body == NULL || bodylen == 0) + return 0; + if (compressed) { + double ratio = estimate_compression_ratio(spooled->spool_source); + bodylen = (size_t)(bodylen * ratio); + } + return bodylen; } else { - result = 0; - SMARTLIST_FOREACH(fps, const char *, digest, { - cached_dir_t *dir = lookup_cached_dir_by_fp(digest); - if (dir) - result += compressed ? dir->dir_z_len : dir->dir_len; - }); + cached_dir_t *cached; + if (spooled->consensus_cache_entry) { + return spooled->cce_len; + } + if (spooled->cached_dir_ref) { + cached = spooled->cached_dir_ref; + } else { + cached = spooled_resource_lookup_cached_dir(spooled, + published_out); + } + if (cached == NULL) { + return 0; + } + size_t result = compressed ? cached->dir_compressed_len : cached->dir_len; + return result; } - return result; } -/** Given a list of microdescriptor hashes, guess how many bytes will be - * needed to transmit them, and return the guess. */ -size_t -dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) -{ - size_t result = smartlist_len(fps) * microdesc_average_size(NULL); - if (compressed) - result /= 2; - return result; -} +/** Return code for spooled_resource_flush_some */ +typedef enum { + SRFS_ERR = -1, + SRFS_MORE = 0, + SRFS_DONE +} spooled_resource_flush_status_t; -/** When we're spooling data onto our outbuf, add more whenever we dip - * below this threshold. */ -#define DIRSERV_BUFFER_MIN 16384 +/** Flush some or all of the bytes from <b>spooled</b> onto <b>conn</b>. + * Return SRFS_ERR on error, SRFS_MORE if there are more bytes to flush from + * this spooled resource, or SRFS_DONE if we are done flushing this spooled + * resource. + */ +static spooled_resource_flush_status_t +spooled_resource_flush_some(spooled_resource_t *spooled, + dir_connection_t *conn) +{ + if (spooled->spool_eagerly) { + /* Spool_eagerly resources are sent all-at-once. */ + const uint8_t *body = NULL; + size_t bodylen = 0; + int r = spooled_resource_lookup_body(spooled, + connection_dir_is_encrypted(conn), + &body, &bodylen, NULL); + if (r == -1 || body == NULL || bodylen == 0) { + /* Absent objects count as "done". */ + return SRFS_DONE; + } + if (conn->compress_state) { + connection_write_to_buf_compress((const char*)body, bodylen, conn, 0); + } else { + connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn)); + } + return SRFS_DONE; + } else { + cached_dir_t *cached = spooled->cached_dir_ref; + consensus_cache_entry_t *cce = spooled->consensus_cache_entry; + if (cached == NULL && cce == NULL) { + /* The cached_dir_t hasn't been materialized yet. So let's look it up. */ + cached = spooled->cached_dir_ref = + spooled_resource_lookup_cached_dir(spooled, NULL); + if (!cached) { + /* Absent objects count as done. */ + return SRFS_DONE; + } + ++cached->refcnt; + tor_assert_nonfatal(spooled->cached_dir_offset == 0); + } -/** Spooling helper: called when we have no more data to spool to <b>conn</b>. - * Flushes any remaining data to be (un)compressed, and changes the spool - * source to NONE. Returns 0 on success, negative on failure. */ -static int -connection_dirserv_finish_spooling(dir_connection_t *conn) -{ - if (conn->zlib_state) { - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; + if (BUG(!cached && !cce)) + return SRFS_DONE; + + int64_t total_len; + const char *ptr; + if (cached) { + total_len = cached->dir_compressed_len; + ptr = cached->dir_compressed; + } else { + total_len = spooled->cce_len; + ptr = (const char *)spooled->cce_body; + } + /* How many bytes left to flush? */ + int64_t remaining; + remaining = total_len - spooled->cached_dir_offset; + if (BUG(remaining < 0)) + return SRFS_ERR; + ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); + if (conn->compress_state) { + connection_write_to_buf_compress( + ptr + spooled->cached_dir_offset, + bytes, conn, 0); + } else { + connection_write_to_buf(ptr + spooled->cached_dir_offset, + bytes, TO_CONN(conn)); + } + spooled->cached_dir_offset += bytes; + if (spooled->cached_dir_offset >= (off_t)total_len) { + return SRFS_DONE; + } else { + return SRFS_MORE; + } } - conn->dir_spool_src = DIR_SPOOL_NONE; - return 0; } -/** Spooling helper: called when we're sending a bunch of server descriptors, - * and the outbuf has become too empty. Pulls some entries from - * fingerprint_stack, and writes the corresponding servers onto outbuf. If we - * run out of entries, flushes the zlib state and sets the spool source to - * NONE. Returns 0 on success, negative on failure. +/** Helper: find the cached_dir_t for a spooled_resource_t, for + * sending it to <b>conn</b>. Set *<b>published_out</b>, if provided, + * to the published time of the cached_dir_t. + * + * DOES NOT increase the reference count on the result. Callers must do that + * themselves if they mean to hang on to it. */ +static cached_dir_t * +spooled_resource_lookup_cached_dir(const spooled_resource_t *spooled, + time_t *published_out) +{ + tor_assert(spooled->spool_eagerly == 0); + cached_dir_t *d = lookup_cached_dir_by_fp(spooled->digest); + if (d != NULL) { + if (published_out) + *published_out = d->published; + } + return d; +} + +/** Helper: Look up the body for an eagerly-served spooled_resource. If + * <b>conn_is_encrypted</b> is false, don't look up any resource that + * shouldn't be sent over an unencrypted connection. On success, set + * <b>body_out</b>, <b>size_out</b>, and <b>published_out</b> to refer + * to the resource's body, size, and publication date, and return 0. + * On failure return -1. */ static int -connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) +spooled_resource_lookup_body(const spooled_resource_t *spooled, + int conn_is_encrypted, + const uint8_t **body_out, + size_t *size_out, + time_t *published_out) { - int by_fp = (conn->dir_spool_src == DIR_SPOOL_SERVER_BY_FP || - conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP); - int extra = (conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP || - conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_DIGEST); - time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH; + tor_assert(spooled->spool_eagerly == 1); - const or_options_t *options = get_options(); + const signed_descriptor_t *sd = NULL; - while (smartlist_len(conn->fingerprint_stack) && - connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { - const char *body; - char *fp = smartlist_pop_last(conn->fingerprint_stack); - const signed_descriptor_t *sd = NULL; - if (by_fp) { - sd = get_signed_descriptor_by_fp(fp, extra, publish_cutoff); - } else { - sd = extra ? extrainfo_get_by_descriptor_digest(fp) - : router_get_by_descriptor_digest(fp); + switch (spooled->spool_source) { + case DIR_SPOOL_EXTRA_BY_FP: { + sd = get_signed_descriptor_by_fp(spooled->digest, 1); + break; } - tor_free(fp); - if (!sd) - continue; - if (!connection_dir_is_encrypted(conn) && !sd->send_unencrypted) { - /* we did this check once before (so we could have an accurate size - * estimate and maybe send a 404 if somebody asked for only bridges on a - * connection), but we need to do it again in case a previously - * unknown bridge descriptor has shown up between then and now. */ - continue; + case DIR_SPOOL_SERVER_BY_FP: { + sd = get_signed_descriptor_by_fp(spooled->digest, 0); + break; } - - /** If we are the bridge authority and the descriptor is a bridge - * descriptor, remember that we served this descriptor for desc stats. */ - if (options->BridgeAuthoritativeDir && by_fp) { - const routerinfo_t *router = - router_get_by_id_digest(sd->identity_digest); - /* router can be NULL here when the bridge auth is asked for its own - * descriptor. */ - if (router && router->purpose == ROUTER_PURPOSE_BRIDGE) - rep_hist_note_desc_served(sd->identity_digest); + case DIR_SPOOL_SERVER_BY_DIGEST: { + sd = router_get_by_descriptor_digest((const char *)spooled->digest); + break; + } + case DIR_SPOOL_EXTRA_BY_DIGEST: { + sd = extrainfo_get_by_descriptor_digest((const char *)spooled->digest); + break; } - body = signed_descriptor_get_body(sd); - if (conn->zlib_state) { - int last = ! smartlist_len(conn->fingerprint_stack); - connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, - last); - if (last) { - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; + case DIR_SPOOL_MICRODESC: { + microdesc_t *md = microdesc_cache_lookup_by_digest256( + get_microdesc_cache(), + (const char *)spooled->digest); + if (! md || ! md->body) { + return -1; } - } else { - connection_write_to_buf(body, - sd->signed_descriptor_len, - TO_CONN(conn)); + *body_out = (const uint8_t *)md->body; + *size_out = md->bodylen; + if (published_out) + *published_out = TIME_MAX; + return 0; } + case DIR_SPOOL_NETWORKSTATUS: + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + return -1; + /* LCOV_EXCL_STOP */ } - if (!smartlist_len(conn->fingerprint_stack)) { - /* We just wrote the last one; finish up. */ - if (conn->zlib_state) { - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; - } - conn->dir_spool_src = DIR_SPOOL_NONE; - smartlist_free(conn->fingerprint_stack); - conn->fingerprint_stack = NULL; - } - return 0; -} + /* If we get here, then we tried to set "sd" to a signed_descriptor_t. */ -/** Spooling helper: called when we're sending a bunch of microdescriptors, - * and the outbuf has become too empty. Pulls some entries from - * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If - * we run out of entries, flushes the zlib state and sets the spool source to - * NONE. Returns 0 on success, negative on failure. - */ -static int -connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) -{ - microdesc_cache_t *cache = get_microdesc_cache(); - while (smartlist_len(conn->fingerprint_stack) && - connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { - char *fp256 = smartlist_pop_last(conn->fingerprint_stack); - microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); - tor_free(fp256); - if (!md || !md->body) - continue; - if (conn->zlib_state) { - int last = !smartlist_len(conn->fingerprint_stack); - connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); - if (last) { - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; - } - } else { - connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); - } + if (sd == NULL) { + return -1; } - if (!smartlist_len(conn->fingerprint_stack)) { - if (conn->zlib_state) { - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; - } - conn->dir_spool_src = DIR_SPOOL_NONE; - smartlist_free(conn->fingerprint_stack); - conn->fingerprint_stack = NULL; + if (sd->send_unencrypted == 0 && ! conn_is_encrypted) { + /* we did this check once before (so we could have an accurate size + * estimate and maybe send a 404 if somebody asked for only bridges on + * a connection), but we need to do it again in case a previously + * unknown bridge descriptor has shown up between then and now. */ + return -1; } + *body_out = (const uint8_t *) signed_descriptor_get_body(sd); + *size_out = sd->signed_descriptor_len; + if (published_out) + *published_out = sd->published_on; return 0; } -/** Spooling helper: Called when we're sending a directory or networkstatus, - * and the outbuf has become too empty. Pulls some bytes from - * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and - * puts them on the outbuf. If we run out of entries, flushes the zlib state - * and sets the spool source to NONE. Returns 0 on success, negative on - * failure. */ -static int -connection_dirserv_add_dir_bytes_to_outbuf(dir_connection_t *conn) -{ - ssize_t bytes; - int64_t remaining; - - bytes = DIRSERV_BUFFER_MIN - connection_get_outbuf_len(TO_CONN(conn)); - tor_assert(bytes > 0); - tor_assert(conn->cached_dir); - if (bytes < 8192) - bytes = 8192; - remaining = conn->cached_dir->dir_z_len - conn->cached_dir_offset; - if (BUG(remaining < 0)) { - remaining = 0; - } - if (bytes > remaining) { - bytes = (ssize_t) remaining; - if (BUG(bytes < 0)) - return -1; +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ +static cached_dir_t * +lookup_cached_dir_by_fp(const uint8_t *fp) +{ + cached_dir_t *d = NULL; + if (tor_digest_is_zero((const char *)fp) && cached_consensuses) { + d = strmap_get(cached_consensuses, "ns"); + } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses) { + /* this here interface is a nasty hack: we're shoving a flavor into + * a digest field. */ + d = strmap_get(cached_consensuses, (const char *)fp); } + return d; +} - if (conn->zlib_state) { - connection_write_to_buf_zlib( - conn->cached_dir->dir_z + conn->cached_dir_offset, - bytes, conn, bytes == remaining); - } else { - connection_write_to_buf(conn->cached_dir->dir_z + conn->cached_dir_offset, - bytes, TO_CONN(conn)); +/** Try to guess the number of bytes that will be needed to send the + * spooled objects for <b>conn</b>'s outgoing spool. In the process, + * remove every element of the spool that refers to an absent object, or + * which was published earlier than <b>cutoff</b>. Set *<b>size_out</b> + * to the number of bytes, and *<b>n_expired_out</b> to the number of + * objects removed for being too old. */ +void +dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, + time_t cutoff, + int compression, + size_t *size_out, + int *n_expired_out) +{ + if (BUG(!conn)) + return; + + smartlist_t *spool = conn->spool; + if (!spool) { + if (size_out) + *size_out = 0; + if (n_expired_out) + *n_expired_out = 0; + return; } - conn->cached_dir_offset += bytes; - if (conn->cached_dir_offset >= (off_t)conn->cached_dir->dir_z_len) { - /* We just wrote the last one; finish up. */ - connection_dirserv_finish_spooling(conn); - cached_dir_decref(conn->cached_dir); - conn->cached_dir = NULL; + int n_expired = 0; + uint64_t total = 0; + SMARTLIST_FOREACH_BEGIN(spool, spooled_resource_t *, spooled) { + time_t published = TIME_MAX; + size_t sz = spooled_resource_estimate_size(spooled, conn, + compression, &published); + if (published < cutoff) { + ++n_expired; + SMARTLIST_DEL_CURRENT(spool, spooled); + spooled_resource_free(spooled); + } else if (sz == 0) { + SMARTLIST_DEL_CURRENT(spool, spooled); + spooled_resource_free(spooled); + } else { + total += sz; + } + } SMARTLIST_FOREACH_END(spooled); + + if (size_out) { + *size_out = (total > SIZE_MAX) ? SIZE_MAX : (size_t)total; } - return 0; + if (n_expired_out) + *n_expired_out = n_expired; } -/** Spooling helper: Called when we're spooling networkstatus objects on - * <b>conn</b>, and the outbuf has become too empty. If the current - * networkstatus object (in <b>conn</b>-\>cached_dir) has more data, pull data - * from there. Otherwise, pop the next fingerprint from fingerprint_stack, - * and start spooling the next networkstatus. (A digest of all 0 bytes is - * treated as a request for the current consensus.) If we run out of entries, - * flushes the zlib state and sets the spool source to NONE. Returns 0 on - * success, negative on failure. */ +/** Helper: used to sort a connection's spool. */ static int -connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) -{ - - while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { - if (conn->cached_dir) { - int uncompressing = (conn->zlib_state != NULL); - int r = connection_dirserv_add_dir_bytes_to_outbuf(conn); - if (conn->dir_spool_src == DIR_SPOOL_NONE) { - /* add_dir_bytes thinks we're done with the cached_dir. But we - * may have more cached_dirs! */ - conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS; - /* This bit is tricky. If we were uncompressing the last - * networkstatus, we may need to make a new zlib object to - * uncompress the next one. */ - if (uncompressing && ! conn->zlib_state && - conn->fingerprint_stack && - smartlist_len(conn->fingerprint_stack)) { - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); - } - } - if (r) return r; - } else if (conn->fingerprint_stack && - smartlist_len(conn->fingerprint_stack)) { - /* Add another networkstatus; start serving it. */ - char *fp = smartlist_pop_last(conn->fingerprint_stack); - cached_dir_t *d = lookup_cached_dir_by_fp(fp); - tor_free(fp); - if (d) { - ++d->refcnt; - conn->cached_dir = d; - conn->cached_dir_offset = 0; - } - } else { - connection_dirserv_finish_spooling(conn); - smartlist_free(conn->fingerprint_stack); - conn->fingerprint_stack = NULL; - return 0; +dirserv_spool_sort_comparison_(const void **a_, const void **b_) +{ + const spooled_resource_t *a = *a_; + const spooled_resource_t *b = *b_; + return fast_memcmp(a->digest, b->digest, sizeof(a->digest)); +} + +/** Sort all the entries in <b>conn</b> by digest. */ +void +dirserv_spool_sort(dir_connection_t *conn) +{ + if (conn->spool == NULL) + return; + smartlist_sort(conn->spool, dirserv_spool_sort_comparison_); +} + +/** Return the cache-info for identity fingerprint <b>fp</b>, or + * its extra-info document if <b>extrainfo</b> is true. Return + * NULL if not found or if the descriptor is older than + * <b>publish_cutoff</b>. */ +static const signed_descriptor_t * +get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo) +{ + if (router_digest_is_me((const char *)fp)) { + if (extrainfo) + return &(router_get_my_extrainfo()->cache_info); + else + return &(router_get_my_routerinfo()->cache_info); + } else { + const routerinfo_t *ri = router_get_by_id_digest((const char *)fp); + if (ri) { + if (extrainfo) + return extrainfo_get_by_descriptor_digest( + ri->cache_info.extra_info_digest); + else + return &ri->cache_info; } } - return 0; + return NULL; } -/** Called whenever we have flushed some directory data in state - * SERVER_WRITING. */ +/** When we're spooling data onto our outbuf, add more whenever we dip + * below this threshold. */ +#define DIRSERV_BUFFER_MIN 16384 + +/** + * Called whenever we have flushed some directory data in state + * SERVER_WRITING, or whenever we want to fill the buffer with initial + * directory data (so that subsequent writes will occur, and trigger this + * function again.) + * + * Return 0 on success, and -1 on failure. + */ int connection_dirserv_flushed_some(dir_connection_t *conn) { tor_assert(conn->base_.state == DIR_CONN_STATE_SERVER_WRITING); - - if (connection_get_outbuf_len(TO_CONN(conn)) >= DIRSERV_BUFFER_MIN) + if (conn->spool == NULL) return 0; - switch (conn->dir_spool_src) { - case DIR_SPOOL_EXTRA_BY_DIGEST: - case DIR_SPOOL_EXTRA_BY_FP: - case DIR_SPOOL_SERVER_BY_DIGEST: - case DIR_SPOOL_SERVER_BY_FP: - return connection_dirserv_add_servers_to_outbuf(conn); - case DIR_SPOOL_MICRODESC: - return connection_dirserv_add_microdescs_to_outbuf(conn); - case DIR_SPOOL_CACHED_DIR: - return connection_dirserv_add_dir_bytes_to_outbuf(conn); - case DIR_SPOOL_NETWORKSTATUS: - return connection_dirserv_add_networkstatus_bytes_to_outbuf(conn); - case DIR_SPOOL_NONE: - default: + while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN && + smartlist_len(conn->spool)) { + spooled_resource_t *spooled = + smartlist_get(conn->spool, smartlist_len(conn->spool)-1); + spooled_resource_flush_status_t status; + status = spooled_resource_flush_some(spooled, conn); + if (status == SRFS_ERR) { + return -1; + } else if (status == SRFS_MORE) { return 0; + } + tor_assert(status == SRFS_DONE); + + /* If we're here, we're done flushing this resource. */ + tor_assert(smartlist_pop_last(conn->spool) == spooled); + spooled_resource_free(spooled); + } + + if (smartlist_len(conn->spool) > 0) { + /* We're still spooling something. */ + return 0; + } + + /* If we get here, we're done. */ + smartlist_free(conn->spool); + conn->spool = NULL; + if (conn->compress_state) { + /* Flush the compression state: there could be more bytes pending in there, + * and we don't want to omit bytes. */ + connection_write_to_buf_compress("", 0, conn, 1); + tor_compress_free(conn->compress_state); + conn->compress_state = NULL; } + return 0; +} + +/** Remove every element from <b>conn</b>'s outgoing spool, and delete + * the spool. */ +void +dir_conn_clear_spool(dir_connection_t *conn) +{ + if (!conn || ! conn->spool) + return; + SMARTLIST_FOREACH(conn->spool, spooled_resource_t *, s, + spooled_resource_free(s)); + smartlist_free(conn->spool); + conn->spool = NULL; } /** Return true iff <b>line</b> is a valid RecommendedPackages line. diff --git a/src/or/dirserv.h b/src/or/dirserv.h index e83da5e5ac..480174d5bb 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,6 +32,61 @@ /** Maximum allowable length of a version line in a networkstatus. */ #define MAX_V_LINE_LEN 128 +/** Ways to convert a spoolable_resource_t to a bunch of bytes. */ +typedef enum dir_spool_source_t { + DIR_SPOOL_SERVER_BY_DIGEST=1, DIR_SPOOL_SERVER_BY_FP, + DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, + DIR_SPOOL_MICRODESC, + DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_CONSENSUS_CACHE_ENTRY, +} dir_spool_source_t; +#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t) + +/** Object to remember the identity of an object that we are spooling, + * or about to spool, in response to a directory request. + * + * (Why do we spool? Because some directory responses are very large, + * and we don't want to just shove the complete answer into the output + * buffer: that would take a ridiculous amount of RAM.) + * + * If the spooled resource is relatively small (like microdescriptors, + * descriptors, etc), we look them up by ID as needed, and add the whole + * thing onto the output buffer at once. If the spooled reseource is + * big (like networkstatus documents), we reference-count it, and add it + * a few K at a time. + */ +typedef struct spooled_resource_t { + /** + * If true, we add the entire object to the outbuf. If false, + * we spool the object a few K at a time. + */ + unsigned spool_eagerly : 1; + /** + * Tells us what kind of object to get, and how to look it up. + */ + dir_spool_source_bitfield_t spool_source : 7; + /** + * Tells us the specific object to spool. + */ + uint8_t digest[DIGEST256_LEN]; + /** + * A large object that we're spooling. Holds a reference count. Only + * used when spool_eagerly is false. + */ + struct cached_dir_t *cached_dir_ref; + /** + * A different kind of large object that we might be spooling. Also + * reference-counted. Also only used when spool_eagerly is false. + */ + struct consensus_cache_entry_t *consensus_cache_entry; + const uint8_t *cce_body; + size_t cce_len; + /** + * The current offset into cached_dir or cce_body. Only used when + * spool_eagerly is false */ + off_t cached_dir_offset; +} spooled_resource_t; + int connection_dirserv_flushed_some(dir_connection_t *conn); int dirserv_add_own_fingerprint(crypto_pk_t *pk); @@ -63,12 +118,13 @@ cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, const char *flavor_name, const common_digests_t *digests, + const uint8_t *sha3_as_signed, time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); -int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, - const char **msg, - int for_unencrypted_conn, - int is_extrainfo); +int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key, + dir_spool_source_t source, + int conn_is_encrytped, + const char **msg_out); int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, const char **msg); void dirserv_orconn_tls_done(const tor_addr_t *addr, @@ -89,13 +145,6 @@ void dirserv_set_node_flags_from_authoritative_status(node_t *node, uint32_t authstatus); int dirserv_would_reject_router(const routerstatus_t *rs); -int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); -int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); -int dirserv_have_any_microdesc(const smartlist_t *fps); -size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, - int compressed); -size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); - char *routerstatus_format_entry( const routerstatus_t *rs, const char *version, @@ -141,5 +190,19 @@ int dirserv_read_measured_bandwidths(const char *from_file, int dirserv_read_guardfraction_file(const char *fname, smartlist_t *vote_routerstatuses); +spooled_resource_t *spooled_resource_new(dir_spool_source_t source, + const uint8_t *digest, + size_t digestlen); +spooled_resource_t *spooled_resource_new_from_cache_entry( + struct consensus_cache_entry_t *entry); +void spooled_resource_free(spooled_resource_t *spooled); +void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, + time_t cutoff, + int compression, + size_t *size_out, + int *n_expired_out); +void dirserv_spool_sort(dir_connection_t *conn); +void dir_conn_clear_spool(dir_connection_t *conn); + #endif diff --git a/src/or/dirvote.c b/src/or/dirvote.c index e92d3b49dc..f5e29eb786 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRVOTE_PRIVATE diff --git a/src/or/dirvote.h b/src/or/dirvote.h index ac7db69db2..e342dc78ea 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dns.c b/src/or/dns.c index 7651501ec3..98b684c904 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -160,8 +160,9 @@ evdns_log_cb(int warn, const char *msg) } if (!strcmpstart(msg, "Nameserver ") && (cp=strstr(msg, " has failed: "))) { char *ns = tor_strndup(msg+11, cp-(msg+11)); - const char *err = strchr(cp, ':')+2; - tor_assert(err); + const char *colon = strchr(cp, ':'); + tor_assert(colon); + const char *err = colon+2; /* Don't warn about a single failed nameserver; we'll warn with 'all * nameservers have failed' if we're completely out of nameservers; * otherwise, the situation is tolerable. */ @@ -522,6 +523,7 @@ send_resolved_cell,(edge_connection_t *conn, uint8_t answer_type, answer_type = RESOLVED_TYPE_ERROR; /* fall through. */ } + /* Falls through. */ case RESOLVED_TYPE_ERROR_TRANSIENT: case RESOLVED_TYPE_ERROR: { @@ -2085,8 +2087,8 @@ assert_cache_ok_(void) #endif -cached_resolve_t -*dns_get_cache_entry(cached_resolve_t *query) +cached_resolve_t * +dns_get_cache_entry(cached_resolve_t *query) { return HT_FIND(cache_map, &cache_root, query); } diff --git a/src/or/dns.h b/src/or/dns.h index 951a2a3467..a81cbd20da 100644 --- a/src/or/dns.h +++ b/src/or/dns.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dns_structs.h b/src/or/dns_structs.h index bc6067213d..dc00e9f7b9 100644 --- a/src/or/dns_structs.h +++ b/src/or/dns_structs.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 8768b2a1d1..54a22a5150 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h index ad0e248c83..6c0643b8dc 100644 --- a/src/or/dnsserv.h +++ b/src/or/dnsserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 5b5e13bced..dc2cab28f7 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -67,7 +67,7 @@ * * While we're building circuits, we track a little "guard state" for * each circuit. We use this to keep track of whether the circuit is - * one that we can use as soon as its done, or whether it's one that + * one that we can use as soon as it's done, or whether it's one that * we should keep around to see if we can do better. In the latter case, * a periodic call to entry_guards_upgrade_waiting_circuits() will * eventually upgrade it. @@ -2073,6 +2073,23 @@ circuit_guard_state_free(circuit_guard_state_t *state) tor_free(state); } +/** Allocate and return a new circuit_guard_state_t to track the result + * of using <b>guard</b> for a given operation. */ +static circuit_guard_state_t * +circuit_guard_state_new(entry_guard_t *guard, unsigned state, + entry_guard_restriction_t *rst) +{ + circuit_guard_state_t *result; + + result = tor_malloc_zero(sizeof(circuit_guard_state_t)); + result->guard = entry_guard_handle_new(guard); + result->state = state; + result->state_set_at = approx_time(); + result->restrictions = rst; + + return result; +} + /** * Pick a suitable entry guard for a circuit in, and place that guard * in *<b>chosen_node_out</b>. Set *<b>guard_state_out</b> to an opaque @@ -2111,11 +2128,7 @@ entry_guard_pick_for_circuit(guard_selection_t *gs, goto fail; *chosen_node_out = node; - *guard_state_out = tor_malloc_zero(sizeof(circuit_guard_state_t)); - (*guard_state_out)->guard = entry_guard_handle_new(guard); - (*guard_state_out)->state = state; - (*guard_state_out)->state_set_at = approx_time(); - (*guard_state_out)->restrictions = rst; + *guard_state_out = circuit_guard_state_new(guard, state, rst); return 0; fail: @@ -2926,6 +2939,32 @@ entry_guard_get_by_id_digest(const char *digest) get_guard_selection_info(), digest); } +/** We are about to connect to bridge with identity <b>digest</b> to fetch its + * descriptor. Create a new guard state for this connection and return it. */ +circuit_guard_state_t * +get_guard_state_for_bridge_desc_fetch(const char *digest) +{ + circuit_guard_state_t *guard_state = NULL; + entry_guard_t *guard = NULL; + + guard = entry_guard_get_by_id_digest_for_guard_selection( + get_guard_selection_info(), digest); + if (!guard) { + return NULL; + } + + /* Update the guard last_tried_to_connect time since it's checked by the + * guard susbsystem. */ + guard->last_tried_to_connect = approx_time(); + + /* Create the guard state */ + guard_state = circuit_guard_state_new(guard, + GUARD_CIRC_STATE_USABLE_ON_COMPLETION, + NULL); + + return guard_state; +} + /** Release all storage held by <b>e</b>. */ STATIC void entry_guard_free(entry_guard_t *e) diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index f02901f5d7..11b618bb50 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -323,6 +323,10 @@ const node_t *guards_choose_dirguard(circuit_guard_state_t **guard_state_out); entry_guard_t *entry_guard_get_by_id_digest_for_guard_selection( guard_selection_t *gs, const char *digest); entry_guard_t *entry_guard_get_by_id_digest(const char *digest); + +circuit_guard_state_t * +get_guard_state_for_bridge_desc_fetch(const char *digest); + void entry_guards_changed_for_guard_selection(guard_selection_t *gs); void entry_guards_changed(void); guard_selection_t * get_guard_selection_info(void); diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index 676adfd8bf..b60d2e55c8 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h index 33d954e8d0..b2cd05db8f 100644 --- a/src/or/ext_orport.h +++ b/src/or/ext_orport.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef EXT_ORPORT_H diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c index eeeb0f1de3..f730106d06 100644 --- a/src/or/fp_pair.c +++ b/src/or/fp_pair.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/fp_pair.h b/src/or/fp_pair.h index b1466581d2..4cea3eda6d 100644 --- a/src/or/fp_pair.h +++ b/src/or/fp_pair.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/geoip.c b/src/or/geoip.c index 2f0047fa06..65d00b8659 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/geoip.h b/src/or/geoip.h index 070296dd07..55ca8ca28c 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hibernate.c b/src/or/hibernate.c index c2b3bbb839..f54109fc90 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -331,7 +331,7 @@ edge_of_accounting_period_containing(time_t now, int get_end) case UNIT_MONTH: { /* If this is before the Nth, we want the Nth of last month. */ if (tm.tm_mday < cfg_start_day || - (tm.tm_mday < cfg_start_day && before)) { + (tm.tm_mday == cfg_start_day && before)) { --tm.tm_mon; } /* Otherwise, the month is correct. */ diff --git a/src/or/hibernate.h b/src/or/hibernate.h index fa9da6de39..8bdb65a927 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c index 43cd8c3258..29681b42b5 100644 --- a/src/or/hs_cache.c +++ b/src/or/hs_cache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h index ba95e73338..ed00424234 100644 --- a/src/or/hs_cache.h +++ b/src/or/hs_cache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_circuitmap.c b/src/or/hs_circuitmap.c index 5003b4b593..ea66fb5194 100644 --- a/src/or/hs_circuitmap.c +++ b/src/or/hs_circuitmap.c @@ -1,11 +1,12 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file hs_circuitmap.c * - * \brief Manage the hidden service circuitmap: A hash table that maps binary - * tokens to introduction and rendezvous circuits. + * \brief Hidden service circuitmap: A hash table that maps binary tokens to + * introduction and rendezvous circuits; it's used both by relays acting as + * intro points and rendezvous points, and also by hidden services themselves. **/ #define HS_CIRCUITMAP_PRIVATE @@ -25,8 +26,8 @@ static struct hs_circuitmap_ht *the_hs_circuitmap = NULL; /* This is a helper function used by the hash table code (HT_). It returns 1 if * two circuits have the same HS token. */ static int -hs_circuits_have_same_token(const or_circuit_t *first_circuit, - const or_circuit_t *second_circuit) +hs_circuits_have_same_token(const circuit_t *first_circuit, + const circuit_t *second_circuit) { const hs_token_t *first_token; const hs_token_t *second_token; @@ -57,7 +58,7 @@ hs_circuits_have_same_token(const or_circuit_t *first_circuit, /* This is a helper function for the hash table code (HT_). It hashes a circuit * HS token into an unsigned int for use as a key by the hash table routines.*/ static inline unsigned int -hs_circuit_hash_token(const or_circuit_t *circuit) +hs_circuit_hash_token(const circuit_t *circuit) { tor_assert(circuit->hs_token); @@ -67,11 +68,11 @@ hs_circuit_hash_token(const or_circuit_t *circuit) /* Register the circuitmap hash table */ HT_PROTOTYPE(hs_circuitmap_ht, // The name of the hashtable struct - or_circuit_t, // The name of the element struct, + circuit_t, // The name of the element struct, hs_circuitmap_node, // The name of HT_ENTRY member hs_circuit_hash_token, hs_circuits_have_same_token) -HT_GENERATE2(hs_circuitmap_ht, or_circuit_t, hs_circuitmap_node, +HT_GENERATE2(hs_circuitmap_ht, circuit_t, hs_circuitmap_node, hs_circuit_hash_token, hs_circuits_have_same_token, 0.6, tor_reallocarray, tor_free_) @@ -116,13 +117,13 @@ hs_token_free(hs_token_t *hs_token) } /** Return the circuit from the circuitmap with token <b>search_token</b>. */ -static or_circuit_t * +static circuit_t * get_circuit_with_token(hs_token_t *search_token) { tor_assert(the_hs_circuitmap); /* We use a dummy circuit object for the hash table search routine. */ - or_circuit_t search_circ; + circuit_t search_circ; search_circ.hs_token = search_token; return HT_FIND(hs_circuitmap_ht, the_hs_circuitmap, &search_circ); } @@ -130,7 +131,7 @@ get_circuit_with_token(hs_token_t *search_token) /* Helper function that registers <b>circ</b> with <b>token</b> on the HS circuitmap. This function steals reference of <b>token</b>. */ static void -hs_circuitmap_register_impl(or_circuit_t *circ, hs_token_t *token) +hs_circuitmap_register_impl(circuit_t *circ, hs_token_t *token) { tor_assert(circ); tor_assert(token); @@ -145,13 +146,12 @@ hs_circuitmap_register_impl(or_circuit_t *circ, hs_token_t *token) take precedence over old ones, so that HSes and clients and reestablish killed circuits without changing the HS token. */ { - or_circuit_t *found_circ; + circuit_t *found_circ; found_circ = get_circuit_with_token(token); if (found_circ) { hs_circuitmap_remove_circuit(found_circ); - if (!found_circ->base_.marked_for_close) { - circuit_mark_for_close(TO_CIRCUIT(found_circ), - END_CIRC_REASON_FINISHED); + if (!found_circ->marked_for_close) { + circuit_mark_for_close(found_circ, END_CIRC_REASON_FINISHED); } } } @@ -165,7 +165,7 @@ hs_circuitmap_register_impl(or_circuit_t *circ, hs_token_t *token) * circuitmap. Use the HS <b>token</b> as the key to the hash table. If * <b>token</b> is not set, clear the circuit of any HS tokens. */ static void -hs_circuitmap_register_circuit(or_circuit_t *circ, +hs_circuitmap_register_circuit(circuit_t *circ, hs_token_type_t type, size_t token_len, const uint8_t *token) { @@ -178,17 +178,19 @@ hs_circuitmap_register_circuit(or_circuit_t *circ, hs_circuitmap_register_impl(circ, hs_token); } -/* Query circuitmap for circuit with <b>token</b> of size <b>token_len</b>. - * Only returns a circuit with purpose equal to the <b>wanted_circ_purpose</b> - * parameter and if it is NOT marked for close. Return NULL if no such circuit - * is found. */ -static or_circuit_t * -hs_circuitmap_get_circuit(hs_token_type_t type, - size_t token_len, - const uint8_t *token, - uint8_t wanted_circ_purpose) +/* Helper function for hs_circuitmap_get_origin_circuit() and + * hs_circuitmap_get_or_circuit(). Because only circuit_t are indexed in the + * circuitmap, this function returns object type so the specialized functions + * using this helper can upcast it to the right type. + * + * Return NULL if not such circuit is found. */ +static circuit_t * +hs_circuitmap_get_circuit_impl(hs_token_type_t type, + size_t token_len, + const uint8_t *token, + uint8_t wanted_circ_purpose) { - or_circuit_t *found_circ = NULL; + circuit_t *found_circ = NULL; tor_assert(the_hs_circuitmap); @@ -202,87 +204,247 @@ hs_circuitmap_get_circuit(hs_token_type_t type, /* Check that the circuit is useful to us */ if (!found_circ || - found_circ->base_.purpose != wanted_circ_purpose || - found_circ->base_.marked_for_close) { + found_circ->purpose != wanted_circ_purpose || + found_circ->marked_for_close) { return NULL; } return found_circ; } -/************** Public circuitmap API ****************************************/ +/* Helper function: Query circuitmap for origin circuit with <b>token</b> of + * size <b>token_len</b> and <b>type</b>. Only returns a circuit with purpose + * equal to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked + * for close. Return NULL if no such circuit is found. */ +static origin_circuit_t * +hs_circuitmap_get_origin_circuit(hs_token_type_t type, + size_t token_len, + const uint8_t *token, + uint8_t wanted_circ_purpose) +{ + circuit_t *circ; + tor_assert(token); + tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose)); -/* Public function: Return v3 introduction circuit with <b>auth_key</b>. Return - * NULL if no such circuit is found in the circuitmap. */ -or_circuit_t * -hs_circuitmap_get_intro_circ_v3(const ed25519_public_key_t *auth_key) + circ = hs_circuitmap_get_circuit_impl(type, token_len, token, + wanted_circ_purpose); + if (!circ) { + return NULL; + } + + tor_assert(CIRCUIT_IS_ORIGIN(circ)); + return TO_ORIGIN_CIRCUIT(circ); +} + +/* Helper function: Query circuitmap for OR circuit with <b>token</b> of size + * <b>token_len</b> and <b>type</b>. Only returns a circuit with purpose equal + * to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked for + * close. Return NULL if no such circuit is found. */ +static or_circuit_t * +hs_circuitmap_get_or_circuit(hs_token_type_t type, + size_t token_len, + const uint8_t *token, + uint8_t wanted_circ_purpose) { - tor_assert(auth_key); + circuit_t *circ; + tor_assert(token); + tor_assert(!CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose)); + + circ = hs_circuitmap_get_circuit_impl(type, token_len, token, + wanted_circ_purpose); + if (!circ) { + return NULL; + } - return hs_circuitmap_get_circuit(HS_TOKEN_INTRO_V3, - ED25519_PUBKEY_LEN, auth_key->pubkey, - CIRCUIT_PURPOSE_INTRO_POINT); + tor_assert(CIRCUIT_IS_ORCIRC(circ)); + return TO_OR_CIRCUIT(circ); } -/* Public function: Return v2 introduction circuit with <b>digest</b>. Return - * NULL if no such circuit is found in the circuitmap. */ +/************** Public circuitmap API ****************************************/ + +/**** Public relay-side getters: */ + +/* Public function: Return a v3 introduction circuit to this relay with + * <b>auth_key</b>. Return NULL if no such circuit is found in the + * circuitmap. */ or_circuit_t * -hs_circuitmap_get_intro_circ_v2(const uint8_t *digest) +hs_circuitmap_get_intro_circ_v3_relay_side( + const ed25519_public_key_t *auth_key) { - tor_assert(digest); + return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V3_RELAY_SIDE, + ED25519_PUBKEY_LEN, auth_key->pubkey, + CIRCUIT_PURPOSE_INTRO_POINT); +} - return hs_circuitmap_get_circuit(HS_TOKEN_INTRO_V2, - REND_TOKEN_LEN, digest, - CIRCUIT_PURPOSE_INTRO_POINT); +/* Public function: Return v2 introduction circuit to this relay with + * <b>digest</b>. Return NULL if no such circuit is found in the circuitmap. */ +or_circuit_t * +hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest) +{ + return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V2_RELAY_SIDE, + REND_TOKEN_LEN, digest, + CIRCUIT_PURPOSE_INTRO_POINT); } -/* Public function: Return rendezvous circuit with rendezvous +/* Public function: Return rendezvous circuit to this relay with rendezvous * <b>cookie</b>. Return NULL if no such circuit is found in the circuitmap. */ or_circuit_t * -hs_circuitmap_get_rend_circ(const uint8_t *cookie) +hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie) { - tor_assert(cookie); - - return hs_circuitmap_get_circuit(HS_TOKEN_REND, - REND_TOKEN_LEN, cookie, - CIRCUIT_PURPOSE_REND_POINT_WAITING); + return hs_circuitmap_get_or_circuit(HS_TOKEN_REND_RELAY_SIDE, + REND_TOKEN_LEN, cookie, + CIRCUIT_PURPOSE_REND_POINT_WAITING); } +/** Public relay-side setters: */ + /* Public function: Register rendezvous circuit with key <b>cookie</b> to the * circuitmap. */ void -hs_circuitmap_register_rend_circ(or_circuit_t *circ, const uint8_t *cookie) +hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ, + const uint8_t *cookie) { - hs_circuitmap_register_circuit(circ, - HS_TOKEN_REND, + hs_circuitmap_register_circuit(TO_CIRCUIT(circ), + HS_TOKEN_REND_RELAY_SIDE, REND_TOKEN_LEN, cookie); } +/* Public function: Register v2 intro circuit with key <b>digest</b> to the + * circuitmap. */ +void +hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ, + const uint8_t *digest) +{ + hs_circuitmap_register_circuit(TO_CIRCUIT(circ), + HS_TOKEN_INTRO_V2_RELAY_SIDE, + REND_TOKEN_LEN, digest); +} + +/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the + * circuitmap. */ +void +hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ, + const ed25519_public_key_t *auth_key) +{ + hs_circuitmap_register_circuit(TO_CIRCUIT(circ), + HS_TOKEN_INTRO_V3_RELAY_SIDE, + ED25519_PUBKEY_LEN, auth_key->pubkey); +} + +/**** Public servide-side getters: */ + +/* Public function: Return v3 introduction circuit with <b>auth_key</b> + * originating from this hidden service. Return NULL if no such circuit is + * found in the circuitmap. */ +origin_circuit_t * +hs_circuitmap_get_intro_circ_v3_service_side(const + ed25519_public_key_t *auth_key) +{ + origin_circuit_t *circ = NULL; + + /* Check first for established intro circuits */ + circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE, + ED25519_PUBKEY_LEN, auth_key->pubkey, + CIRCUIT_PURPOSE_S_INTRO); + if (circ) { + return circ; + } + + /* ...if nothing found, check for pending intro circs */ + circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE, + ED25519_PUBKEY_LEN, auth_key->pubkey, + CIRCUIT_PURPOSE_S_ESTABLISH_INTRO); + + return circ; +} + +/* Public function: Return v2 introduction circuit originating from this hidden + * service with <b>digest</b>. Return NULL if no such circuit is found in the + * circuitmap. */ +origin_circuit_t * +hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest) +{ + origin_circuit_t *circ = NULL; + + /* Check first for established intro circuits */ + circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE, + REND_TOKEN_LEN, digest, + CIRCUIT_PURPOSE_S_INTRO); + if (circ) { + return circ; + } + + /* ...if nothing found, check for pending intro circs */ + circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE, + REND_TOKEN_LEN, digest, + CIRCUIT_PURPOSE_S_ESTABLISH_INTRO); + + return circ; +} + +/* Public function: Return rendezvous circuit originating from this hidden + * service with rendezvous <b>cookie</b>. Return NULL if no such circuit is + * found in the circuitmap. */ +origin_circuit_t * +hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie) +{ + origin_circuit_t *circ = NULL; + + /* Try to check if we have a connecting circuit. */ + circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE, + REND_TOKEN_LEN, cookie, + CIRCUIT_PURPOSE_S_CONNECT_REND); + if (circ) { + return circ; + } + + /* Then try for connected circuit. */ + circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE, + REND_TOKEN_LEN, cookie, + CIRCUIT_PURPOSE_S_REND_JOINED); + return circ; +} + +/**** Public servide-side setters: */ /* Public function: Register v2 intro circuit with key <b>digest</b> to the * circuitmap. */ void -hs_circuitmap_register_intro_circ_v2(or_circuit_t *circ, const uint8_t *digest) +hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ, + const uint8_t *digest) { - hs_circuitmap_register_circuit(circ, - HS_TOKEN_INTRO_V2, + hs_circuitmap_register_circuit(TO_CIRCUIT(circ), + HS_TOKEN_INTRO_V2_SERVICE_SIDE, REND_TOKEN_LEN, digest); } /* Public function: Register v3 intro circuit with key <b>auth_key</b> to the * circuitmap. */ void -hs_circuitmap_register_intro_circ_v3(or_circuit_t *circ, - const ed25519_public_key_t *auth_key) +hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ, + const ed25519_public_key_t *auth_key) { - hs_circuitmap_register_circuit(circ, - HS_TOKEN_INTRO_V3, + hs_circuitmap_register_circuit(TO_CIRCUIT(circ), + HS_TOKEN_INTRO_V3_SERVICE_SIDE, ED25519_PUBKEY_LEN, auth_key->pubkey); } -/** Remove this circuit from the HS circuitmap. Clear its HS token, and remove - * it from the hashtable. */ +/* Public function: Register rendezvous circuit with key <b>cookie</b> to the + * circuitmap. */ +void +hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ, + const uint8_t *cookie) +{ + hs_circuitmap_register_circuit(TO_CIRCUIT(circ), + HS_TOKEN_REND_SERVICE_SIDE, + REND_TOKEN_LEN, cookie); +} + +/**** Misc public functions: */ + +/** Public function: Remove this circuit from the HS circuitmap. Clear its HS + * token, and remove it from the hashtable. */ void -hs_circuitmap_remove_circuit(or_circuit_t *circ) +hs_circuitmap_remove_circuit(circuit_t *circ) { tor_assert(the_hs_circuitmap); @@ -291,14 +453,14 @@ hs_circuitmap_remove_circuit(or_circuit_t *circ) } /* Remove circ from circuitmap */ - or_circuit_t *tmp; + circuit_t *tmp; tmp = HT_REMOVE(hs_circuitmap_ht, the_hs_circuitmap, circ); /* ... and ensure the removal was successful. */ if (tmp) { tor_assert(tmp == circ); } else { log_warn(LD_BUG, "Could not find circuit (%u) in circuitmap.", - circ->p_circ_id); + circ->n_circ_id); } /* Clear token from circ */ @@ -306,7 +468,7 @@ hs_circuitmap_remove_circuit(or_circuit_t *circ) circ->hs_token = NULL; } -/* Initialize the global HS circuitmap. */ +/* Public function: Initialize the global HS circuitmap. */ void hs_circuitmap_init(void) { @@ -316,7 +478,7 @@ hs_circuitmap_init(void) HT_INIT(hs_circuitmap_ht, the_hs_circuitmap); } -/* Free all memory allocated by the global HS circuitmap. */ +/* Public function: Free all memory allocated by the global HS circuitmap. */ void hs_circuitmap_free_all(void) { diff --git a/src/or/hs_circuitmap.h b/src/or/hs_circuitmap.h index b587039310..33d5b64117 100644 --- a/src/or/hs_circuitmap.h +++ b/src/or/hs_circuitmap.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,26 +9,52 @@ #ifndef TOR_HS_CIRCUITMAP_H #define TOR_HS_CIRCUITMAP_H -typedef HT_HEAD(hs_circuitmap_ht, or_circuit_t) hs_circuitmap_ht; +typedef HT_HEAD(hs_circuitmap_ht, circuit_t) hs_circuitmap_ht; typedef struct hs_token_s hs_token_t; struct or_circuit_t; +struct origin_circuit_t; /** Public HS circuitmap API: */ -struct or_circuit_t *hs_circuitmap_get_rend_circ(const uint8_t *cookie); -struct or_circuit_t *hs_circuitmap_get_intro_circ_v3( - const ed25519_public_key_t *auth_key); -struct or_circuit_t *hs_circuitmap_get_intro_circ_v2(const uint8_t *digest); - -void hs_circuitmap_register_rend_circ(struct or_circuit_t *circ, - const uint8_t *cookie); -void hs_circuitmap_register_intro_circ_v2(struct or_circuit_t *circ, - const uint8_t *digest); -void hs_circuitmap_register_intro_circ_v3(struct or_circuit_t *circ, +/** Public relay-side API: */ + +struct or_circuit_t * +hs_circuitmap_get_intro_circ_v3_relay_side(const + ed25519_public_key_t *auth_key); +struct or_circuit_t * +hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest); +struct or_circuit_t * +hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie); + +void hs_circuitmap_register_rend_circ_relay_side(struct or_circuit_t *circ, + const uint8_t *cookie); +void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ, + const uint8_t *digest); +void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ, const ed25519_public_key_t *auth_key); -void hs_circuitmap_remove_circuit(struct or_circuit_t *circ); +/** Public service-side API: */ + +struct origin_circuit_t * +hs_circuitmap_get_intro_circ_v3_service_side(const + ed25519_public_key_t *auth_key); +struct origin_circuit_t * +hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest); +struct origin_circuit_t * +hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie); + +void hs_circuitmap_register_intro_circ_v2_service_side( + struct origin_circuit_t *circ, + const uint8_t *digest); +void hs_circuitmap_register_intro_circ_v3_service_side( + struct origin_circuit_t *circ, + const ed25519_public_key_t *auth_key); +void hs_circuitmap_register_rend_circ_service_side( + struct origin_circuit_t *circ, + const uint8_t *cookie); + +void hs_circuitmap_remove_circuit(struct circuit_t *circ); void hs_circuitmap_init(void); void hs_circuitmap_free_all(void); @@ -37,12 +63,19 @@ void hs_circuitmap_free_all(void); /** Represents the type of HS token. */ typedef enum { - /** A rendezvous cookie (128bit)*/ - HS_TOKEN_REND, - /** A v2 introduction point pubkey (160bit) */ - HS_TOKEN_INTRO_V2, - /** A v3 introduction point pubkey (256bit) */ - HS_TOKEN_INTRO_V3, + /** A rendezvous cookie on a relay (128bit)*/ + HS_TOKEN_REND_RELAY_SIDE, + /** A v2 introduction point pubkey on a relay (160bit) */ + HS_TOKEN_INTRO_V2_RELAY_SIDE, + /** A v3 introduction point pubkey on a relay (256bit) */ + HS_TOKEN_INTRO_V3_RELAY_SIDE, + + /** A rendezvous cookie on a hidden service (128bit)*/ + HS_TOKEN_REND_SERVICE_SIDE, + /** A v2 introduction point pubkey on a hidden service (160bit) */ + HS_TOKEN_INTRO_V2_SERVICE_SIDE, + /** A v3 introduction point pubkey on a hidden service (256bit) */ + HS_TOKEN_INTRO_V3_SERVICE_SIDE, } hs_token_type_t; /** Represents a token used in the HS protocol. Each such token maps to a diff --git a/src/or/hs_common.c b/src/or/hs_common.c index de96946ab5..42508126f8 100644 --- a/src/or/hs_common.c +++ b/src/or/hs_common.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,6 +9,8 @@ * protocol. **/ +#define HS_COMMON_PRIVATE + #include "or.h" #include "config.h" @@ -16,6 +18,80 @@ #include "hs_common.h" #include "rendcommon.h" +/* Make sure that the directory for <b>service</b> is private, using the config + * <b>username</b>. + * If <b>create</b> is true: + * - if the directory exists, change permissions if needed, + * - if the directory does not exist, create it with the correct permissions. + * If <b>create</b> is false: + * - if the directory exists, check permissions, + * - if the directory does not exist, check if we think we can create it. + * Return 0 on success, -1 on failure. */ +int +hs_check_service_private_dir(const char *username, const char *path, + unsigned int dir_group_readable, + unsigned int create) +{ + cpd_check_t check_opts = CPD_NONE; + + tor_assert(path); + + if (create) { + check_opts |= CPD_CREATE; + } else { + check_opts |= CPD_CHECK_MODE_ONLY; + check_opts |= CPD_CHECK; + } + if (dir_group_readable) { + check_opts |= CPD_GROUP_READ; + } + /* Check/create directory */ + if (check_private_dir(path, check_opts, username) < 0) { + return -1; + } + return 0; +} + +/** Get the default HS time period length in minutes from the consensus. */ +STATIC uint64_t +get_time_period_length(void) +{ + int32_t time_period_length = networkstatus_get_param(NULL, "hsdir-interval", + HS_TIME_PERIOD_LENGTH_DEFAULT, + HS_TIME_PERIOD_LENGTH_MIN, + HS_TIME_PERIOD_LENGTH_MAX); + /* Make sure it's a positive value. */ + tor_assert(time_period_length >= 0); + /* uint64_t will always be able to contain a int32_t */ + return (uint64_t) time_period_length; +} + +/** Get the HS time period number at time <b>now</b> */ +STATIC uint64_t +get_time_period_num(time_t now) +{ + uint64_t time_period_num; + uint64_t time_period_length = get_time_period_length(); + uint64_t minutes_since_epoch = now / 60; + + /* Now subtract half a day to fit the prop224 time period schedule (see + * section [TIME-PERIODS]). */ + tor_assert(minutes_since_epoch > HS_TIME_PERIOD_ROTATION_OFFSET); + minutes_since_epoch -= HS_TIME_PERIOD_ROTATION_OFFSET; + + /* Calculate the time period */ + time_period_num = minutes_since_epoch / time_period_length; + return time_period_num; +} + +/** Get the number of the _upcoming_ HS time period, given that the current + * time is <b>now</b>. */ +uint64_t +hs_get_next_time_period_num(time_t now) +{ + return get_time_period_num(now) + 1; +} + /* Create a new rend_data_t for a specific given <b>version</b>. * Return a pointer to the newly allocated data structure. */ static rend_data_t * diff --git a/src/or/hs_common.h b/src/or/hs_common.h index e0ab510ea4..a8fded652a 100644 --- a/src/or/hs_common.h +++ b/src/or/hs_common.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,12 +17,42 @@ /* Version 3 of the protocol (prop224). */ #define HS_VERSION_THREE 3 -/* Denotes ed25519 authentication key on ESTABLISH_INTRO cell. */ -#define AUTH_KEY_ED25519 0x02 +/** Try to maintain this many intro points per service by default. */ +#define NUM_INTRO_POINTS_DEFAULT 3 +/** Maximum number of intro points per service. */ +#define NUM_INTRO_POINTS_MAX 10 +/** Number of extra intro points we launch if our set of intro nodes is empty. + * See proposal 155, section 4. */ +#define NUM_INTRO_POINTS_EXTRA 2 + +/** If we can't build our intro circuits, don't retry for this long. */ +#define INTRO_CIRC_RETRY_PERIOD (60*5) +/** Don't try to build more than this many circuits before giving up for a + * while.*/ +#define MAX_INTRO_CIRCS_PER_PERIOD 10 +/** How many times will a hidden service operator attempt to connect to a + * requested rendezvous point before giving up? */ +#define MAX_REND_FAILURES 1 +/** How many seconds should we spend trying to connect to a requested + * rendezvous point before giving up? */ +#define MAX_REND_TIMEOUT 30 /* String prefix for the signature of ESTABLISH_INTRO */ #define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1" +/* The default HS time period length */ +#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */ +/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */ +/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */ +/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */ + +int hs_check_service_private_dir(const char *username, const char *path, + unsigned int dir_group_readable, + unsigned int create); + void rend_data_free(rend_data_t *data); rend_data_t *rend_data_dup(const rend_data_t *data); rend_data_t *rend_data_client_create(const char *onion_address, @@ -39,5 +69,18 @@ const char *rend_data_get_desc_id(const rend_data_t *rend_data, const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out); +uint64_t hs_get_next_time_period_num(time_t now); + +#ifdef HS_COMMON_PRIVATE + +#ifdef TOR_UNIT_TESTS + +STATIC uint64_t get_time_period_length(void); +STATIC uint64_t get_time_period_num(time_t now); + +#endif /* TOR_UNIT_TESTS */ + +#endif /* HS_COMMON_PRIVATE */ + #endif /* TOR_HS_COMMON_H */ diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c index 938b7a77df..2a000f5002 100644 --- a/src/or/hs_descriptor.c +++ b/src/or/hs_descriptor.c @@ -1,9 +1,55 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file hs_descriptor.c * \brief Handle hidden service descriptor encoding/decoding. + * + * \details + * Here is a graphical depiction of an HS descriptor and its layers: + * + * +------------------------------------------------------+ + * |DESCRIPTOR HEADER: | + * | hs-descriptor 3 | + * | descriptor-lifetime 180 | + * | ... | + * | superencrypted | + * |+---------------------------------------------------+ | + * ||SUPERENCRYPTED LAYER (aka OUTER ENCRYPTED LAYER): | | + * || desc-auth-type x25519 | | + * || desc-auth-ephemeral-key | | + * || auth-client | | + * || auth-client | | + * || ... | | + * || encrypted | | + * ||+-------------------------------------------------+| | + * |||ENCRYPTED LAYER (aka INNER ENCRYPTED LAYER): || | + * ||| create2-formats || | + * ||| intro-auth-required || | + * ||| introduction-point || | + * ||| introduction-point || | + * ||| ... || | + * ||+-------------------------------------------------+| | + * |+---------------------------------------------------+ | + * +------------------------------------------------------+ + * + * The DESCRIPTOR HEADER section is completely unencrypted and contains generic + * descriptor metadata. + * + * The SUPERENCRYPTED LAYER section is the first layer of encryption, and it's + * encrypted using the blinded public key of the hidden service to protect + * against entities who don't know its onion address. The clients of the hidden + * service know its onion address and blinded public key, whereas third-parties + * (like HSDirs) don't know it (except if it's a public hidden service). + * + * The ENCRYPTED LAYER section is the second layer of encryption, and it's + * encrypted using the client authorization key material (if those exist). When + * client authorization is enabled, this second layer of encryption protects + * the descriptor content from unauthorized entities. If client authorization + * is disabled, this second layer of encryption does not provide any extra + * security but is still present. The plaintext of this layer contains all the + * information required to connect to the hidden service like its list of + * introduction points. **/ /* For unit tests.*/ @@ -23,29 +69,36 @@ #define str_desc_cert "descriptor-signing-key-cert" #define str_rev_counter "revision-counter" #define str_superencrypted "superencrypted" +#define str_encrypted "encrypted" #define str_signature "signature" #define str_lifetime "descriptor-lifetime" /* Constant string value for the encrypted part of the descriptor. */ #define str_create2_formats "create2-formats" -#define str_auth_required "authentication-required" +#define str_intro_auth_required "intro-auth-required" #define str_single_onion "single-onion-service" #define str_intro_point "introduction-point" #define str_ip_auth_key "auth-key" #define str_ip_enc_key "enc-key" -#define str_ip_enc_key_cert "enc-key-certification" +#define str_ip_enc_key_cert "enc-key-cert" +#define str_ip_legacy_key "legacy-key" +#define str_ip_legacy_key_cert "legacy-key-cert" #define str_intro_point_start "\n" str_intro_point " " /* Constant string value for the construction to encrypt the encrypted data * section. */ -#define str_enc_hsdir_data "hsdir-superencrypted-data" +#define str_enc_const_superencryption "hsdir-superencrypted-data" +#define str_enc_const_encryption "hsdir-encrypted-data" /* Prefix required to compute/verify HS desc signatures */ #define str_desc_sig_prefix "Tor onion service descriptor sig v3" +#define str_desc_auth_type "desc-auth-type" +#define str_desc_auth_key "desc-auth-ephemeral-key" +#define str_desc_auth_client "auth-client" +#define str_encrypted "encrypted" /* Authentication supported types. */ static const struct { hs_desc_auth_type_t type; const char *identifier; -} auth_types[] = { - { HS_DESC_AUTH_PASSWORD, "password" }, +} intro_auth_types[] = { { HS_DESC_AUTH_ED25519, "ed25519" }, /* Indicate end of array. */ { 0, NULL } @@ -62,10 +115,19 @@ static token_rule_t hs_desc_v3_token_table[] = { END_OF_TABLE }; +/* Descriptor ruleset for the superencrypted section. */ +static token_rule_t hs_desc_superencrypted_v3_token_table[] = { + T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ), + T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ), + T1N(str_desc_auth_client, R3_DESC_AUTH_CLIENT, GE(3), NO_OBJ), + T1(str_encrypted, R3_ENCRYPTED, NO_ARGS, NEED_OBJ), + END_OF_TABLE +}; + /* Descriptor ruleset for the encrypted section. */ static token_rule_t hs_desc_encrypted_v3_token_table[] = { T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ), - T01(str_auth_required, R3_AUTHENTICATION_REQUIRED, ARGS, NO_OBJ), + T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, ARGS, NO_OBJ), T01(str_single_onion, R3_SINGLE_ONION_SERVICE, ARGS, NO_OBJ), END_OF_TABLE }; @@ -74,9 +136,10 @@ static token_rule_t hs_desc_encrypted_v3_token_table[] = { static token_rule_t hs_desc_intro_point_v3_token_table[] = { T1_START(str_intro_point, R3_INTRODUCTION_POINT, EQ(1), NO_OBJ), T1(str_ip_auth_key, R3_INTRO_AUTH_KEY, NO_ARGS, NEED_OBJ), - T1(str_ip_enc_key, R3_INTRO_ENC_KEY, ARGS, OBJ_OK), - T1_END(str_ip_enc_key_cert, R3_INTRO_ENC_KEY_CERTIFICATION, - NO_ARGS, NEED_OBJ), + T1(str_ip_enc_key, R3_INTRO_ENC_KEY, GE(2), OBJ_OK), + T1(str_ip_enc_key_cert, R3_INTRO_ENC_KEY_CERT, ARGS, OBJ_OK), + T01(str_ip_legacy_key, R3_INTRO_LEGACY_KEY, ARGS, NEED_KEY_1024), + T01(str_ip_legacy_key_cert, R3_INTRO_LEGACY_KEY_CERT, ARGS, OBJ_OK), END_OF_TABLE }; @@ -93,8 +156,12 @@ desc_intro_point_free(hs_desc_intro_point_t *ip) smartlist_free(ip->link_specifiers); } tor_cert_free(ip->auth_key_cert); - if (ip->enc_key_type == HS_DESC_KEY_TYPE_LEGACY) { - crypto_pk_free(ip->enc_key.legacy); + tor_cert_free(ip->enc_key_cert); + if (ip->legacy.key) { + crypto_pk_free(ip->legacy.key); + } + if (ip->legacy.cert.encoded) { + tor_free(ip->legacy.cert.encoded); } tor_free(ip); } @@ -107,8 +174,8 @@ desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc) return; } - if (desc->encrypted_blob) { - tor_free(desc->encrypted_blob); + if (desc->superencrypted_blob) { + tor_free(desc->superencrypted_blob); } tor_cert_free(desc->signing_key_cert); @@ -123,9 +190,9 @@ desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc) return; } - if (desc->auth_types) { - SMARTLIST_FOREACH(desc->auth_types, char *, a, tor_free(a)); - smartlist_free(desc->auth_types); + if (desc->intro_auth_types) { + SMARTLIST_FOREACH(desc->intro_auth_types, char *, a, tor_free(a)); + smartlist_free(desc->intro_auth_types); } if (desc->intro_points) { SMARTLIST_FOREACH(desc->intro_points, hs_desc_intro_point_t *, ip, @@ -135,6 +202,135 @@ desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc) memwipe(desc, 0, sizeof(*desc)); } +/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out. + * We use SHA3-256 for the MAC computation. + * This function can't fail. */ +static void +build_mac(const uint8_t *mac_key, size_t mac_key_len, + const uint8_t *salt, size_t salt_len, + const uint8_t *encrypted, size_t encrypted_len, + uint8_t *mac_out, size_t mac_len) +{ + crypto_digest_t *digest; + + const uint64_t mac_len_netorder = tor_htonll(mac_key_len); + const uint64_t salt_len_netorder = tor_htonll(salt_len); + + tor_assert(mac_key); + tor_assert(salt); + tor_assert(encrypted); + tor_assert(mac_out); + + digest = crypto_digest256_new(DIGEST_SHA3_256); + /* As specified in section 2.5 of proposal 224, first add the mac key + * then add the salt first and then the encrypted section. */ + + crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8); + crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len); + crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8); + crypto_digest_add_bytes(digest, (const char *) salt, salt_len); + crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len); + crypto_digest_get_digest(digest, (char *) mac_out, mac_len); + crypto_digest_free(digest); +} + +/* Using a given decriptor object, build the secret input needed for the + * KDF and put it in the dst pointer which is an already allocated buffer + * of size dstlen. */ +static void +build_secret_input(const hs_descriptor_t *desc, uint8_t *dst, size_t dstlen) +{ + size_t offset = 0; + + tor_assert(desc); + tor_assert(dst); + tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN <= dstlen); + + /* XXX use the destination length as the memcpy length */ + /* Copy blinded public key. */ + memcpy(dst, desc->plaintext_data.blinded_pubkey.pubkey, + sizeof(desc->plaintext_data.blinded_pubkey.pubkey)); + offset += sizeof(desc->plaintext_data.blinded_pubkey.pubkey); + /* Copy subcredential. */ + memcpy(dst + offset, desc->subcredential, sizeof(desc->subcredential)); + offset += sizeof(desc->subcredential); + /* Copy revision counter value. */ + set_uint64(dst + offset, tor_ntohll(desc->plaintext_data.revision_counter)); + offset += sizeof(uint64_t); + tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN == offset); +} + +/* Do the KDF construction and put the resulting data in key_out which is of + * key_out_len length. It uses SHAKE-256 as specified in the spec. */ +static void +build_kdf_key(const hs_descriptor_t *desc, + const uint8_t *salt, size_t salt_len, + uint8_t *key_out, size_t key_out_len, + int is_superencrypted_layer) +{ + uint8_t secret_input[HS_DESC_ENCRYPTED_SECRET_INPUT_LEN]; + crypto_xof_t *xof; + + tor_assert(desc); + tor_assert(salt); + tor_assert(key_out); + + /* Build the secret input for the KDF computation. */ + build_secret_input(desc, secret_input, sizeof(secret_input)); + + xof = crypto_xof_new(); + /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */ + crypto_xof_add_bytes(xof, secret_input, sizeof(secret_input)); + crypto_xof_add_bytes(xof, salt, salt_len); + + /* Feed in the right string constant based on the desc layer */ + if (is_superencrypted_layer) { + crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_superencryption, + strlen(str_enc_const_superencryption)); + } else { + crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_encryption, + strlen(str_enc_const_encryption)); + } + + /* Eat from our KDF. */ + crypto_xof_squeeze_bytes(xof, key_out, key_out_len); + crypto_xof_free(xof); + memwipe(secret_input, 0, sizeof(secret_input)); +} + +/* Using the given descriptor and salt, run it through our KDF function and + * then extract a secret key in key_out, the IV in iv_out and MAC in mac_out. + * This function can't fail. */ +static void +build_secret_key_iv_mac(const hs_descriptor_t *desc, + const uint8_t *salt, size_t salt_len, + uint8_t *key_out, size_t key_len, + uint8_t *iv_out, size_t iv_len, + uint8_t *mac_out, size_t mac_len, + int is_superencrypted_layer) +{ + size_t offset = 0; + uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN]; + + tor_assert(desc); + tor_assert(salt); + tor_assert(key_out); + tor_assert(iv_out); + tor_assert(mac_out); + + build_kdf_key(desc, salt, salt_len, kdf_key, sizeof(kdf_key), + is_superencrypted_layer); + /* Copy the bytes we need for both the secret key and IV. */ + memcpy(key_out, kdf_key, key_len); + offset += key_len; + memcpy(iv_out, kdf_key + offset, iv_len); + offset += iv_len; + memcpy(mac_out, kdf_key + offset, mac_len); + /* Extra precaution to make sure we are not out of bound. */ + tor_assert((offset + mac_len) == sizeof(kdf_key)); + memwipe(kdf_key, 0, sizeof(kdf_key)); +} + /* === ENCODING === */ /* Encode the given link specifier objects into a newly allocated string. @@ -217,101 +413,68 @@ encode_link_specifiers(const smartlist_t *specs) return encoded_b64; } -/* Encode an introduction point encryption key and return a newly allocated - * string with it. On failure, return NULL. */ +/* Encode an introduction point legacy key and certificate. Return a newly + * allocated string with it. On failure, return NULL. */ static char * -encode_enc_key(const ed25519_public_key_t *sig_key, - const hs_desc_intro_point_t *ip) +encode_legacy_key(const hs_desc_intro_point_t *ip) { - char *encoded = NULL; - time_t now = time(NULL); + char *key_str, b64_cert[256], *encoded = NULL; + size_t key_str_len; - tor_assert(sig_key); tor_assert(ip); - switch (ip->enc_key_type) { - case HS_DESC_KEY_TYPE_LEGACY: - { - char *key_str, b64_cert[256]; - ssize_t cert_len; - size_t key_str_len; - uint8_t *cert_data = NULL; - - /* Create cross certification cert. */ - cert_len = tor_make_rsa_ed25519_crosscert(sig_key, ip->enc_key.legacy, - now + HS_DESC_CERT_LIFETIME, - &cert_data); - if (cert_len < 0) { - log_warn(LD_REND, "Unable to create legacy crosscert."); - goto err; - } - /* Encode cross cert. */ - if (base64_encode(b64_cert, sizeof(b64_cert), (const char *) cert_data, - cert_len, BASE64_ENCODE_MULTILINE) < 0) { - tor_free(cert_data); - log_warn(LD_REND, "Unable to encode legacy crosscert."); - goto err; - } - tor_free(cert_data); - /* Convert the encryption key to a string. */ - if (crypto_pk_write_public_key_to_string(ip->enc_key.legacy, &key_str, - &key_str_len) < 0) { - log_warn(LD_REND, "Unable to encode legacy encryption key."); - goto err; - } - tor_asprintf(&encoded, - "%s legacy\n%s" /* Newline is added by the call above. */ - "%s\n" - "-----BEGIN CROSSCERT-----\n" - "%s" - "-----END CROSSCERT-----", - str_ip_enc_key, key_str, - str_ip_enc_key_cert, b64_cert); - tor_free(key_str); - break; - } - case HS_DESC_KEY_TYPE_CURVE25519: - { - int signbit, ret; - char *encoded_cert, key_fp_b64[CURVE25519_BASE64_PADDED_LEN + 1]; - ed25519_keypair_t curve_kp; + /* Encode cross cert. */ + if (base64_encode(b64_cert, sizeof(b64_cert), + (const char *) ip->legacy.cert.encoded, + ip->legacy.cert.len, BASE64_ENCODE_MULTILINE) < 0) { + log_warn(LD_REND, "Unable to encode legacy crosscert."); + goto done; + } + /* Convert the encryption key to PEM format NUL terminated. */ + if (crypto_pk_write_public_key_to_string(ip->legacy.key, &key_str, + &key_str_len) < 0) { + log_warn(LD_REND, "Unable to encode legacy encryption key."); + goto done; + } + tor_asprintf(&encoded, + "%s \n%s" /* Newline is added by the call above. */ + "%s\n" + "-----BEGIN CROSSCERT-----\n" + "%s" + "-----END CROSSCERT-----", + str_ip_legacy_key, key_str, + str_ip_legacy_key_cert, b64_cert); + tor_free(key_str); - if (ed25519_keypair_from_curve25519_keypair(&curve_kp, &signbit, - &ip->enc_key.curve25519)) { - goto err; - } - tor_cert_t *cross_cert = tor_cert_create(&curve_kp, - CERT_TYPE_CROSS_HS_IP_KEYS, - sig_key, now, - HS_DESC_CERT_LIFETIME, - CERT_FLAG_INCLUDE_SIGNING_KEY); - memwipe(&curve_kp, 0, sizeof(curve_kp)); - if (!cross_cert) { - goto err; - } - ret = tor_cert_encode_ed22519(cross_cert, &encoded_cert); - tor_cert_free(cross_cert); - if (ret) { - goto err; - } - if (curve25519_public_to_base64(key_fp_b64, - &ip->enc_key.curve25519.pubkey) < 0) { - tor_free(encoded_cert); - goto err; - } - tor_asprintf(&encoded, - "%s ntor %s\n" - "%s\n%s", - str_ip_enc_key, key_fp_b64, - str_ip_enc_key_cert, encoded_cert); - tor_free(encoded_cert); - break; + done: + return encoded; +} + +/* Encode an introduction point encryption key and certificate. Return a newly + * allocated string with it. On failure, return NULL. */ +static char * +encode_enc_key(const hs_desc_intro_point_t *ip) +{ + char *encoded = NULL, *encoded_cert; + char key_b64[CURVE25519_BASE64_PADDED_LEN + 1]; + + tor_assert(ip); + + /* Base64 encode the encryption key for the "enc-key" field. */ + if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) { + goto done; } - default: - tor_assert(0); + if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) { + goto done; } + tor_asprintf(&encoded, + "%s ntor %s\n" + "%s\n%s", + str_ip_enc_key, key_b64, + str_ip_enc_key_cert, encoded_cert); + tor_free(encoded_cert); - err: + done: return encoded; } @@ -346,7 +509,7 @@ encode_intro_point(const ed25519_public_key_t *sig_key, /* Encryption key encoding. */ { - char *encoded_enc_key = encode_enc_key(sig_key, ip); + char *encoded_enc_key = encode_enc_key(ip); if (encoded_enc_key == NULL) { goto err; } @@ -354,6 +517,18 @@ encode_intro_point(const ed25519_public_key_t *sig_key, tor_free(encoded_enc_key); } + /* Legacy key if any. */ + if (ip->legacy.key != NULL) { + /* Strong requirement else the IP creation was badly done. */ + tor_assert(ip->legacy.cert.encoded); + char *encoded_legacy_key = encode_legacy_key(ip); + if (encoded_legacy_key == NULL) { + goto err; + } + smartlist_add_asprintf(lines, "%s", encoded_legacy_key); + tor_free(encoded_legacy_key); + } + /* Join them all in one blob of text. */ encoded_ip = smartlist_join_strings(lines, "\n", 1, NULL); @@ -363,142 +538,23 @@ encode_intro_point(const ed25519_public_key_t *sig_key, return encoded_ip; } -/* Using a given decriptor object, build the secret input needed for the - * KDF and put it in the dst pointer which is an already allocated buffer - * of size dstlen. */ -static void -build_secret_input(const hs_descriptor_t *desc, uint8_t *dst, size_t dstlen) -{ - size_t offset = 0; - - tor_assert(desc); - tor_assert(dst); - tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN <= dstlen); - - /* XXX use the destination length as the memcpy length */ - /* Copy blinded public key. */ - memcpy(dst, desc->plaintext_data.blinded_pubkey.pubkey, - sizeof(desc->plaintext_data.blinded_pubkey.pubkey)); - offset += sizeof(desc->plaintext_data.blinded_pubkey.pubkey); - /* Copy subcredential. */ - memcpy(dst + offset, desc->subcredential, sizeof(desc->subcredential)); - offset += sizeof(desc->subcredential); - /* Copy revision counter value. */ - set_uint64(dst + offset, tor_ntohll(desc->plaintext_data.revision_counter)); - offset += sizeof(uint64_t); - tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN == offset); -} - -/* Do the KDF construction and put the resulting data in key_out which is of - * key_out_len length. It uses SHAKE-256 as specified in the spec. */ -static void -build_kdf_key(const hs_descriptor_t *desc, - const uint8_t *salt, size_t salt_len, - uint8_t *key_out, size_t key_out_len) -{ - uint8_t secret_input[HS_DESC_ENCRYPTED_SECRET_INPUT_LEN]; - crypto_xof_t *xof; - - tor_assert(desc); - tor_assert(salt); - tor_assert(key_out); - - /* Build the secret input for the KDF computation. */ - build_secret_input(desc, secret_input, sizeof(secret_input)); - - xof = crypto_xof_new(); - /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */ - crypto_xof_add_bytes(xof, secret_input, sizeof(secret_input)); - crypto_xof_add_bytes(xof, salt, salt_len); - crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_hsdir_data, - strlen(str_enc_hsdir_data)); - /* Eat from our KDF. */ - crypto_xof_squeeze_bytes(xof, key_out, key_out_len); - crypto_xof_free(xof); - memwipe(secret_input, 0, sizeof(secret_input)); -} - -/* Using the given descriptor and salt, run it through our KDF function and - * then extract a secret key in key_out, the IV in iv_out and MAC in mac_out. - * This function can't fail. */ -static void -build_secret_key_iv_mac(const hs_descriptor_t *desc, - const uint8_t *salt, size_t salt_len, - uint8_t *key_out, size_t key_len, - uint8_t *iv_out, size_t iv_len, - uint8_t *mac_out, size_t mac_len) -{ - size_t offset = 0; - uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN]; - - tor_assert(desc); - tor_assert(salt); - tor_assert(key_out); - tor_assert(iv_out); - tor_assert(mac_out); - - build_kdf_key(desc, salt, salt_len, kdf_key, sizeof(kdf_key)); - /* Copy the bytes we need for both the secret key and IV. */ - memcpy(key_out, kdf_key, key_len); - offset += key_len; - memcpy(iv_out, kdf_key + offset, iv_len); - offset += iv_len; - memcpy(mac_out, kdf_key + offset, mac_len); - /* Extra precaution to make sure we are not out of bound. */ - tor_assert((offset + mac_len) == sizeof(kdf_key)); - memwipe(kdf_key, 0, sizeof(kdf_key)); -} - -/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out. - * We use SHA3-256 for the MAC computation. - * This function can't fail. */ -static void -build_mac(const uint8_t *mac_key, size_t mac_key_len, - const uint8_t *salt, size_t salt_len, - const uint8_t *encrypted, size_t encrypted_len, - uint8_t *mac_out, size_t mac_len) -{ - crypto_digest_t *digest; - - const uint64_t mac_len_netorder = tor_htonll(mac_key_len); - const uint64_t salt_len_netorder = tor_htonll(salt_len); - - tor_assert(mac_key); - tor_assert(salt); - tor_assert(encrypted); - tor_assert(mac_out); - - digest = crypto_digest256_new(DIGEST_SHA3_256); - /* As specified in section 2.5 of proposal 224, first add the mac key - * then add the salt first and then the encrypted section. */ - - crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8); - crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len); - crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8); - crypto_digest_add_bytes(digest, (const char *) salt, salt_len); - crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len); - crypto_digest_get_digest(digest, (char *) mac_out, mac_len); - crypto_digest_free(digest); -} - /* Given a source length, return the new size including padding for the * plaintext encryption. */ static size_t compute_padded_plaintext_length(size_t plaintext_len) { size_t plaintext_padded_len; + const int padding_block_length = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE; /* Make sure we won't overflow. */ - tor_assert(plaintext_len <= - (SIZE_T_CEILING - HS_DESC_PLAINTEXT_PADDING_MULTIPLE)); - - /* Get the extra length we need to add. For example, if srclen is 234 bytes, - * this will expand to (2 * 128) == 256 thus an extra 22 bytes. */ - plaintext_padded_len = CEIL_DIV(plaintext_len, - HS_DESC_PLAINTEXT_PADDING_MULTIPLE) * - HS_DESC_PLAINTEXT_PADDING_MULTIPLE; + tor_assert(plaintext_len <= (SIZE_T_CEILING - padding_block_length)); + + /* Get the extra length we need to add. For example, if srclen is 10200 + * bytes, this will expand to (2 * 10k) == 20k thus an extra 9800 bytes. */ + plaintext_padded_len = CEIL_DIV(plaintext_len, padding_block_length) * + padding_block_length; /* Can never be extra careful. Make sure we are _really_ padded. */ - tor_assert(!(plaintext_padded_len % HS_DESC_PLAINTEXT_PADDING_MULTIPLE)); + tor_assert(!(plaintext_padded_len % padding_block_length)); return plaintext_padded_len; } @@ -530,7 +586,8 @@ build_plaintext_padding(const char *plaintext, size_t plaintext_len, * data. Return size of the encrypted data buffer. */ static size_t build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext, - size_t plaintext_len, uint8_t **encrypted_out) + size_t plaintext_len, uint8_t **encrypted_out, + int is_superencrypted_layer) { size_t encrypted_len; uint8_t *padded_plaintext, *encrypted; @@ -541,15 +598,21 @@ build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext, tor_assert(plaintext); tor_assert(encrypted_out); + /* If we are encrypting the middle layer of the descriptor, we need to first + pad the plaintext */ + if (is_superencrypted_layer) { + encrypted_len = build_plaintext_padding(plaintext, plaintext_len, + &padded_plaintext); + /* Extra precautions that we have a valid padding length. */ + tor_assert(!(encrypted_len % HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE)); + } else { /* No padding required for inner layers */ + padded_plaintext = tor_memdup(plaintext, plaintext_len); + encrypted_len = plaintext_len; + } + /* This creates a cipher for AES. It can't fail. */ cipher = crypto_cipher_new_with_iv_and_bits(key, iv, HS_DESC_ENCRYPTED_BIT_SIZE); - /* This can't fail. */ - encrypted_len = build_plaintext_padding(plaintext, plaintext_len, - &padded_plaintext); - /* Extra precautions that we have a valie padding length. */ - tor_assert(encrypted_len <= HS_DESC_PADDED_PLAINTEXT_MAX_LEN); - tor_assert(!(encrypted_len % HS_DESC_PLAINTEXT_PADDING_MULTIPLE)); /* We use a stream cipher so the encrypted length will be the same as the * plaintext padded length. */ encrypted = tor_malloc_zero(encrypted_len); @@ -563,12 +626,13 @@ build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext, return encrypted_len; } -/* Encrypt the given plaintext buffer and using the descriptor to get the +/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> to get the * keys. Set encrypted_out with the encrypted data and return the length of - * it. */ + * it. <b>is_superencrypted_layer</b> is set if this is the outer encrypted + * layer of the descriptor. */ static size_t encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext, - char **encrypted_out) + char **encrypted_out, int is_superencrypted_layer) { char *final_blob; size_t encrypted_len, final_blob_len, offset = 0; @@ -589,11 +653,13 @@ encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext, build_secret_key_iv_mac(desc, salt, sizeof(salt), secret_key, sizeof(secret_key), secret_iv, sizeof(secret_iv), - mac_key, sizeof(mac_key)); + mac_key, sizeof(mac_key), + is_superencrypted_layer); /* Build the encrypted part that is do the actual encryption. */ encrypted_len = build_encrypted(secret_key, secret_iv, plaintext, - strlen(plaintext), &encrypted); + strlen(plaintext), &encrypted, + is_superencrypted_layer); memwipe(secret_key, 0, sizeof(secret_key)); memwipe(secret_iv, 0, sizeof(secret_iv)); /* This construction is specified in section 2.5 of proposal 224. */ @@ -625,20 +691,89 @@ encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext, return final_blob_len; } -/* Take care of encoding the encrypted data section and then encrypting it - * with the descriptor's key. A newly allocated NUL terminated string pointer - * containing the encrypted encoded blob is put in encrypted_blob_out. Return - * 0 on success else a negative value. */ -static int -encode_encrypted_data(const hs_descriptor_t *desc, - char **encrypted_blob_out) +/* Create and return a string containing a fake client-auth entry. It's the + * responsibility of the caller to free the returned string. This function will + * never fail. */ +static char * +get_fake_auth_client_str(void) { - int ret = -1; - char *encoded_str, *encrypted_blob; - smartlist_t *lines = smartlist_new(); + char *auth_client_str = NULL; + /* We are gonna fill these arrays with fake base64 data. They are all double + * the size of their binary representation to fit the base64 overhead. */ + char client_id_b64[8*2]; + char iv_b64[16*2]; + char encrypted_cookie_b64[16*2]; + int retval; + + /* This is a macro to fill a field with random data and then base64 it. */ +#define FILL_WITH_FAKE_DATA_AND_BASE64(field) STMT_BEGIN \ + crypto_rand((char *)field, sizeof(field)); \ + retval = base64_encode_nopad(field##_b64, sizeof(field##_b64), \ + field, sizeof(field)); \ + tor_assert(retval > 0); \ + STMT_END + + { /* Get those fakes! */ + uint8_t client_id[8]; /* fake client-id */ + uint8_t iv[16]; /* fake IV (initialization vector) */ + uint8_t encrypted_cookie[16]; /* fake encrypted cookie */ + + FILL_WITH_FAKE_DATA_AND_BASE64(client_id); + FILL_WITH_FAKE_DATA_AND_BASE64(iv); + FILL_WITH_FAKE_DATA_AND_BASE64(encrypted_cookie); + } + + /* Build the final string */ + tor_asprintf(&auth_client_str, "%s %s %s %s", str_desc_auth_client, + client_id_b64, iv_b64, encrypted_cookie_b64); + +#undef FILL_WITH_FAKE_DATA_AND_BASE64 + + return auth_client_str; +} - tor_assert(desc); - tor_assert(encrypted_blob_out); +/** How many lines of "client-auth" we want in our descriptors; fake or not. */ +#define CLIENT_AUTH_ENTRIES_BLOCK_SIZE 16 + +/** Create the "client-auth" part of the descriptor and return a + * newly-allocated string with it. It's the responsibility of the caller to + * free the returned string. */ +static char * +get_fake_auth_client_lines(void) +{ + /* XXX: Client authorization is still not implemented, so all this function + does is make fake clients */ + int i = 0; + smartlist_t *auth_client_lines = smartlist_new(); + char *auth_client_lines_str = NULL; + + /* Make a line for each fake client */ + const int num_fake_clients = CLIENT_AUTH_ENTRIES_BLOCK_SIZE; + for (i = 0; i < num_fake_clients; i++) { + char *auth_client_str = get_fake_auth_client_str(); + tor_assert(auth_client_str); + smartlist_add(auth_client_lines, auth_client_str); + } + + /* Join all lines together to form final string */ + auth_client_lines_str = smartlist_join_strings(auth_client_lines, + "\n", 1, NULL); + /* Cleanup the mess */ + SMARTLIST_FOREACH(auth_client_lines, char *, a, tor_free(a)); + smartlist_free(auth_client_lines); + + return auth_client_lines_str; +} + +/* Create the inner layer of the descriptor (which includes the intro points, + * etc.). Return a newly-allocated string with the layer plaintext, or NULL if + * an error occured. It's the responsibility of the caller to free the returned + * string. */ +static char * +get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc) +{ + char *encoded_str = NULL; + smartlist_t *lines = smartlist_new(); /* Build the start of the section prior to the introduction points. */ { @@ -649,12 +784,12 @@ encode_encrypted_data(const hs_descriptor_t *desc, smartlist_add_asprintf(lines, "%s %d\n", str_create2_formats, ONION_HANDSHAKE_TYPE_NTOR); - if (desc->encrypted_data.auth_types && - smartlist_len(desc->encrypted_data.auth_types)) { + if (desc->encrypted_data.intro_auth_types && + smartlist_len(desc->encrypted_data.intro_auth_types)) { /* Put the authentication-required line. */ - char *buf = smartlist_join_strings(desc->encrypted_data.auth_types, " ", - 0, NULL); - smartlist_add_asprintf(lines, "%s %s\n", str_auth_required, buf); + char *buf = smartlist_join_strings(desc->encrypted_data.intro_auth_types, + " ", 0, NULL); + smartlist_add_asprintf(lines, "%s %s\n", str_intro_auth_required, buf); tor_free(buf); } @@ -679,31 +814,159 @@ encode_encrypted_data(const hs_descriptor_t *desc, * then encrypt it. */ encoded_str = smartlist_join_strings(lines, "", 0, NULL); - /* Encrypt the section into an encrypted blob that we'll base64 encode - * before returning it. */ + err: + SMARTLIST_FOREACH(lines, char *, l, tor_free(l)); + smartlist_free(lines); + + return encoded_str; +} + +/* Create the middle layer of the descriptor, which includes the client auth + * data and the encrypted inner layer (provided as a base64 string at + * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the + * layer plaintext, or NULL if an error occured. It's the responsibility of the + * caller to free the returned string. */ +static char * +get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc, + const char *layer2_b64_ciphertext) +{ + char *layer1_str = NULL; + smartlist_t *lines = smartlist_new(); + + /* XXX: Disclaimer: This function generates only _fake_ client auth + * data. Real client auth is not yet implemented, but client auth data MUST + * always be present in descriptors. In the future this function will be + * refactored to use real client auth data if they exist (#20700). */ + (void) *desc; + + /* Specify auth type */ + smartlist_add_asprintf(lines, "%s %s\n", str_desc_auth_type, "x25519"); + + { /* Create fake ephemeral x25519 key */ + char fake_key_base64[CURVE25519_BASE64_PADDED_LEN + 1]; + curve25519_keypair_t fake_x25519_keypair; + if (curve25519_keypair_generate(&fake_x25519_keypair, 0) < 0) { + goto done; + } + if (curve25519_public_to_base64(fake_key_base64, + &fake_x25519_keypair.pubkey) < 0) { + goto done; + } + smartlist_add_asprintf(lines, "%s %s\n", + str_desc_auth_key, fake_key_base64); + /* No need to memwipe any of these fake keys. They will go unused. */ + } + + { /* Create fake auth-client lines. */ + char *auth_client_lines = get_fake_auth_client_lines(); + tor_assert(auth_client_lines); + smartlist_add(lines, auth_client_lines); + } + + /* create encrypted section */ { - char *enc_b64; - ssize_t enc_b64_len, ret_len, enc_len; + smartlist_add_asprintf(lines, + "%s\n" + "-----BEGIN MESSAGE-----\n" + "%s" + "-----END MESSAGE-----", + str_encrypted, layer2_b64_ciphertext); + } - enc_len = encrypt_descriptor_data(desc, encoded_str, &encrypted_blob); - tor_free(encoded_str); - /* Get the encoded size plus a NUL terminating byte. */ - enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1; - enc_b64 = tor_malloc_zero(enc_b64_len); - /* Base64 the encrypted blob before returning it. */ - ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len, - BASE64_ENCODE_MULTILINE); - /* Return length doesn't count the NUL byte. */ - tor_assert(ret_len == (enc_b64_len - 1)); - tor_free(encrypted_blob); - *encrypted_blob_out = enc_b64; + layer1_str = smartlist_join_strings(lines, "", 0, NULL); + + done: + SMARTLIST_FOREACH(lines, char *, a, tor_free(a)); + smartlist_free(lines); + + return layer1_str; +} + +/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before + * returning it. <b>desc</b> is provided to derive the encryption + * keys. <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the + * middle (superencrypted) layer of the descriptor. It's the responsibility of + * the caller to free the returned string. */ +static char * +encrypt_desc_data_and_base64(const hs_descriptor_t *desc, + const char *encoded_str, + int is_superencrypted_layer) +{ + char *enc_b64; + ssize_t enc_b64_len, ret_len, enc_len; + char *encrypted_blob = NULL; + + enc_len = encrypt_descriptor_data(desc, encoded_str, &encrypted_blob, + is_superencrypted_layer); + /* Get the encoded size plus a NUL terminating byte. */ + enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1; + enc_b64 = tor_malloc_zero(enc_b64_len); + /* Base64 the encrypted blob before returning it. */ + ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len, + BASE64_ENCODE_MULTILINE); + /* Return length doesn't count the NUL byte. */ + tor_assert(ret_len == (enc_b64_len - 1)); + tor_free(encrypted_blob); + + return enc_b64; +} + +/* Generate and encode the superencrypted portion of <b>desc</b>. This also + * involves generating the encrypted portion of the descriptor, and performing + * the superencryption. A newly allocated NUL-terminated string pointer + * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0 + * on success else a negative value. */ +static int +encode_superencrypted_data(const hs_descriptor_t *desc, + char **encrypted_blob_out) +{ + int ret = -1; + char *layer2_str = NULL; + char *layer2_b64_ciphertext = NULL; + char *layer1_str = NULL; + char *layer1_b64_ciphertext = NULL; + + tor_assert(desc); + tor_assert(encrypted_blob_out); + + /* Func logic: We first create the inner layer of the descriptor (layer2). + * We then encrypt it and use it to create the middle layer of the descriptor + * (layer1). Finally we superencrypt the middle layer and return it to our + * caller. */ + + /* Create inner descriptor layer */ + layer2_str = get_inner_encrypted_layer_plaintext(desc); + if (!layer2_str) { + goto err; } + + /* Encrypt and b64 the inner layer */ + layer2_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer2_str, 0); + if (!layer2_b64_ciphertext) { + goto err; + } + + /* Now create middle descriptor layer given the inner layer */ + layer1_str = get_outer_encrypted_layer_plaintext(desc,layer2_b64_ciphertext); + if (!layer1_str) { + goto err; + } + + /* Encrypt and base64 the middle layer */ + layer1_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer1_str, 1); + if (!layer1_b64_ciphertext) { + goto err; + } + /* Success! */ ret = 0; err: - SMARTLIST_FOREACH(lines, char *, l, tor_free(l)); - smartlist_free(lines); + tor_free(layer1_str); + tor_free(layer2_str); + tor_free(layer2_b64_ciphertext); + + *encrypted_blob_out = layer1_b64_ciphertext; return ret; } @@ -756,7 +1019,7 @@ desc_encode_v3(const hs_descriptor_t *desc, /* Build the superencrypted data section. */ { char *enc_b64_blob=NULL; - if (encode_encrypted_data(desc, &enc_b64_blob) < 0) { + if (encode_superencrypted_data(desc, &enc_b64_blob) < 0) { goto err; } smartlist_add_asprintf(lines, @@ -796,6 +1059,13 @@ desc_encode_v3(const hs_descriptor_t *desc, encoded_str = smartlist_join_strings(lines, "\n", 1, NULL); *encoded_out = encoded_str; + if (strlen(encoded_str) >= hs_cache_get_max_descriptor_size()) { + log_warn(LD_GENERAL, "We just made an HS descriptor that's too big (%d)." + "Failing.", (int)strlen(encoded_str)); + tor_free(encoded_str); + goto err; + } + /* XXX: Trigger a control port event. */ /* Success! */ @@ -894,14 +1164,14 @@ decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list) tor_assert(desc); tor_assert(list); - desc->auth_types = smartlist_new(); - smartlist_split_string(desc->auth_types, list, " ", 0, 0); + desc->intro_auth_types = smartlist_new(); + smartlist_split_string(desc->intro_auth_types, list, " ", 0, 0); /* Validate the types that we at least know about one. */ - SMARTLIST_FOREACH_BEGIN(desc->auth_types, const char *, auth) { - for (int idx = 0; auth_types[idx].identifier; idx++) { - if (!strncmp(auth, auth_types[idx].identifier, - strlen(auth_types[idx].identifier))) { + SMARTLIST_FOREACH_BEGIN(desc->intro_auth_types, const char *, auth) { + for (int idx = 0; intro_auth_types[idx].identifier; idx++) { + if (!strncmp(auth, intro_auth_types[idx].identifier, + strlen(intro_auth_types[idx].identifier))) { match = 1; break; } @@ -1024,7 +1294,7 @@ STATIC int encrypted_data_length_is_valid(size_t len) { /* Make sure there is enough data for the salt and the mac. The equality is - * there to ensure that there is at least one byte of encrypted data. */ + there to ensure that there is at least one byte of encrypted data. */ if (len <= HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN) { log_warn(LD_REND, "Length of descriptor's encrypted data is too small. " "Got %lu but minimum value is %d", @@ -1037,12 +1307,17 @@ encrypted_data_length_is_valid(size_t len) return 0; } -/* Decrypt the encrypted section of the descriptor using the given descriptor - * object desc. A newly allocated NUL terminated string is put in - * decrypted_out. Return the length of decrypted_out on success else 0 is - * returned and decrypted_out is set to NULL. */ +/** Decrypt an encrypted descriptor layer at <b>encrypted_blob</b> of size + * <b>encrypted_blob_size</b>. Use the descriptor object <b>desc</b> to + * generate the right decryption keys; set <b>decrypted_out</b> to the + * plaintext. If <b>is_superencrypted_layer</b> is set, this is the outter + * encrypted layer of the descriptor. */ static size_t -desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) +decrypt_desc_layer(const hs_descriptor_t *desc, + const uint8_t *encrypted_blob, + size_t encrypted_blob_size, + int is_superencrypted_layer, + char **decrypted_out) { uint8_t *decrypted = NULL; uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN]; @@ -1052,41 +1327,33 @@ desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) tor_assert(decrypted_out); tor_assert(desc); - tor_assert(desc->plaintext_data.encrypted_blob); + tor_assert(encrypted_blob); - /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC */ - if (!encrypted_data_length_is_valid( - desc->plaintext_data.encrypted_blob_size)) { + /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC . + * Make sure we have enough space for all these things. */ + if (!encrypted_data_length_is_valid(encrypted_blob_size)) { goto err; } /* Start of the blob thus the salt. */ - salt = desc->plaintext_data.encrypted_blob; + salt = encrypted_blob; + /* Next is the encrypted data. */ - encrypted = desc->plaintext_data.encrypted_blob + - HS_DESC_ENCRYPTED_SALT_LEN; - encrypted_len = desc->plaintext_data.encrypted_blob_size - + encrypted = encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN; + encrypted_len = encrypted_blob_size - (HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN); + tor_assert(encrypted_len > 0); /* guaranteed by the check above */ - /* At the very end is the MAC. Make sure it's of the right size. */ - { - desc_mac = encrypted + encrypted_len; - size_t desc_mac_size = desc->plaintext_data.encrypted_blob_size - - (desc_mac - desc->plaintext_data.encrypted_blob); - if (desc_mac_size != DIGEST256_LEN) { - log_warn(LD_REND, "Service descriptor MAC length of encrypted data " - "is invalid (%lu, expected %u)", - (unsigned long) desc_mac_size, DIGEST256_LEN); - goto err; - } - } + /* And last comes the MAC. */ + desc_mac = encrypted_blob + encrypted_blob_size - DIGEST256_LEN; /* KDF construction resulting in a key from which the secret key, IV and MAC * key are extracted which is what we need for the decryption. */ build_secret_key_iv_mac(desc, salt, HS_DESC_ENCRYPTED_SALT_LEN, secret_key, sizeof(secret_key), secret_iv, sizeof(secret_iv), - mac_key, sizeof(mac_key)); + mac_key, sizeof(mac_key), + is_superencrypted_layer); /* Build MAC. */ build_mac(mac_key, sizeof(mac_key), salt, HS_DESC_ENCRYPTED_SALT_LEN, @@ -1116,7 +1383,7 @@ desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) } { - /* Adjust length to remove NULL padding bytes */ + /* Adjust length to remove NUL padding bytes */ uint8_t *end = memchr(decrypted, 0, encrypted_len); result_len = encrypted_len; if (end) { @@ -1142,6 +1409,222 @@ desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) return result_len; } +/* Basic validation that the superencrypted client auth portion of the + * descriptor is well-formed and recognized. Return True if so, otherwise + * return False. */ +static int +superencrypted_auth_data_is_valid(smartlist_t *tokens) +{ + /* XXX: This is just basic validation for now. When we implement client auth, + we can refactor this function so that it actually parses and saves the + data. */ + + { /* verify desc auth type */ + const directory_token_t *tok; + tok = find_by_keyword(tokens, R3_DESC_AUTH_TYPE); + tor_assert(tok->n_args >= 1); + if (strcmp(tok->args[0], "x25519")) { + log_warn(LD_DIR, "Unrecognized desc auth type"); + return 0; + } + } + + { /* verify desc auth key */ + const directory_token_t *tok; + curve25519_public_key_t k; + tok = find_by_keyword(tokens, R3_DESC_AUTH_KEY); + tor_assert(tok->n_args >= 1); + if (curve25519_public_from_base64(&k, tok->args[0]) < 0) { + log_warn(LD_DIR, "Bogus desc auth key in HS desc"); + return 0; + } + } + + /* verify desc auth client items */ + SMARTLIST_FOREACH_BEGIN(tokens, const directory_token_t *, tok) { + if (tok->tp == R3_DESC_AUTH_CLIENT) { + tor_assert(tok->n_args >= 3); + } + } SMARTLIST_FOREACH_END(tok); + + return 1; +} + +/* Parse <b>message</b>, the plaintext of the superencrypted portion of an HS + * descriptor. Set <b>encrypted_out</b> to the encrypted blob, and return its + * size */ +STATIC size_t +decode_superencrypted(const char *message, size_t message_len, + uint8_t **encrypted_out) +{ + int retval = 0; + memarea_t *area = NULL; + smartlist_t *tokens = NULL; + + area = memarea_new(); + tokens = smartlist_new(); + if (tokenize_string(area, message, message + message_len, tokens, + hs_desc_superencrypted_v3_token_table, 0) < 0) { + log_warn(LD_REND, "Superencrypted portion is not parseable"); + goto err; + } + + /* Do some rudimentary validation of the authentication data */ + if (!superencrypted_auth_data_is_valid(tokens)) { + log_warn(LD_REND, "Invalid auth data"); + goto err; + } + + /* Extract the encrypted data section. */ + { + const directory_token_t *tok; + tok = find_by_keyword(tokens, R3_ENCRYPTED); + tor_assert(tok->object_body); + if (strcmp(tok->object_type, "MESSAGE") != 0) { + log_warn(LD_REND, "Desc superencrypted data section is invalid"); + goto err; + } + /* Make sure the length of the encrypted blob is valid. */ + if (!encrypted_data_length_is_valid(tok->object_size)) { + goto err; + } + + /* Copy the encrypted blob to the descriptor object so we can handle it + * latter if needed. */ + tor_assert(tok->object_size <= INT_MAX); + *encrypted_out = tor_memdup(tok->object_body, tok->object_size); + retval = (int) tok->object_size; + } + + err: + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); + smartlist_free(tokens); + if (area) { + memarea_drop_all(area); + } + + return retval; +} + +/* Decrypt both the superencrypted and the encrypted section of the descriptor + * using the given descriptor object <b>desc</b>. A newly allocated NUL + * terminated string is put in decrypted_out which contains the inner encrypted + * layer of the descriptor. Return the length of decrypted_out on success else + * 0 is returned and decrypted_out is set to NULL. */ +static size_t +desc_decrypt_all(const hs_descriptor_t *desc, char **decrypted_out) +{ + size_t decrypted_len = 0; + size_t encrypted_len = 0; + size_t superencrypted_len = 0; + char *superencrypted_plaintext = NULL; + uint8_t *encrypted_blob = NULL; + + /** Function logic: This function takes us from the descriptor header to the + * inner encrypted layer, by decrypting and decoding the middle descriptor + * layer. In the end we return the contents of the inner encrypted layer to + * our caller. */ + + /* 1. Decrypt middle layer of descriptor */ + superencrypted_len = decrypt_desc_layer(desc, + desc->plaintext_data.superencrypted_blob, + desc->plaintext_data.superencrypted_blob_size, + 1, + &superencrypted_plaintext); + if (!superencrypted_len) { + log_warn(LD_REND, "Decrypting superencrypted desc failed."); + goto err; + } + tor_assert(superencrypted_plaintext); + + /* 2. Parse "superencrypted" */ + encrypted_len = decode_superencrypted(superencrypted_plaintext, + superencrypted_len, + &encrypted_blob); + if (!encrypted_len) { + log_warn(LD_REND, "Decrypting encrypted desc failed."); + goto err; + } + tor_assert(encrypted_blob); + + /* 3. Decrypt "encrypted" and set decrypted_out */ + char *decrypted_desc; + decrypted_len = decrypt_desc_layer(desc, + encrypted_blob, encrypted_len, + 0, &decrypted_desc); + if (!decrypted_len) { + log_warn(LD_REND, "Decrypting encrypted desc failed."); + goto err; + } + tor_assert(decrypted_desc); + + *decrypted_out = decrypted_desc; + + err: + tor_free(superencrypted_plaintext); + tor_free(encrypted_blob); + + return decrypted_len; +} + +/* Given the token tok for an intro point legacy key, the list of tokens, the + * introduction point ip being decoded and the descriptor desc from which it + * comes from, decode the legacy key and set the intro point object. Return 0 + * on success else -1 on failure. */ +static int +decode_intro_legacy_key(const directory_token_t *tok, + smartlist_t *tokens, + hs_desc_intro_point_t *ip, + const hs_descriptor_t *desc) +{ + tor_assert(tok); + tor_assert(tokens); + tor_assert(ip); + tor_assert(desc); + + if (!crypto_pk_public_exponent_ok(tok->key)) { + log_warn(LD_REND, "Introduction point legacy key is invalid"); + goto err; + } + ip->legacy.key = crypto_pk_dup_key(tok->key); + /* Extract the legacy cross certification cert which MUST be present if we + * have a legacy key. */ + tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY_CERT); + if (!tok) { + log_warn(LD_REND, "Introduction point legacy key cert is missing"); + goto err; + } + tor_assert(tok->object_body); + if (strcmp(tok->object_type, "CROSSCERT")) { + /* Info level because this might be an unknown field that we should + * ignore. */ + log_info(LD_REND, "Introduction point legacy encryption key " + "cross-certification has an unknown format."); + goto err; + } + /* Keep a copy of the certificate. */ + ip->legacy.cert.encoded = tor_memdup(tok->object_body, tok->object_size); + ip->legacy.cert.len = tok->object_size; + /* The check on the expiration date is for the entire lifetime of a + * certificate which is 24 hours. However, a descriptor has a maximum + * lifetime of 12 hours meaning we have a 12h difference between the two + * which ultimately accomodate the clock skewed client. */ + if (rsa_ed25519_crosscert_check(ip->legacy.cert.encoded, + ip->legacy.cert.len, ip->legacy.key, + &desc->plaintext_data.signing_pubkey, + approx_time() - HS_DESC_CERT_LIFETIME)) { + log_warn(LD_REND, "Unable to check cross-certification on the " + "introduction point legacy encryption key."); + ip->cross_certified = 0; + goto err; + } + + /* Success. */ + return 0; + err: + return -1; +} + /* Given the start of a section and the end of it, decode a single * introduction point from that section. Return a newly allocated introduction * point object containing the decoded data. Return NULL if the section can't @@ -1152,7 +1635,6 @@ decode_introduction_point(const hs_descriptor_t *desc, const char *start) hs_desc_intro_point_t *ip = NULL; memarea_t *area = NULL; smartlist_t *tokens = NULL; - tor_cert_t *cross_cert = NULL; const directory_token_t *tok; tor_assert(desc); @@ -1186,84 +1668,67 @@ decode_introduction_point(const hs_descriptor_t *desc, const char *start) log_warn(LD_REND, "Unexpected object type for introduction auth key"); goto err; } - /* Parse cert and do some validation. */ if (cert_parse_and_validate(&ip->auth_key_cert, tok->object_body, tok->object_size, CERT_TYPE_AUTH_HS_IP_KEY, "introduction point auth-key") < 0) { goto err; } + /* Validate authentication certificate with descriptor signing key. */ + if (tor_cert_checksig(ip->auth_key_cert, + &desc->plaintext_data.signing_pubkey, 0) < 0) { + log_warn(LD_REND, "Invalid authentication key signature"); + goto err; + } - /* Exactly one "enc-key" ... */ + /* Exactly one "enc-key" SP "ntor" SP key NL */ tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY); if (!strcmp(tok->args[0], "ntor")) { - /* "enc-key" SP "ntor" SP key NL */ - if (tok->n_args != 2 || tok->object_body) { - log_warn(LD_REND, "Introduction point ntor encryption key is invalid"); - goto err; - } + /* This field is using GE(2) so for possible forward compatibility, we + * accept more fields but must be at least 2. */ + tor_assert(tok->n_args >= 2); - if (curve25519_public_from_base64(&ip->enc_key.curve25519.pubkey, - tok->args[1]) < 0) { - log_warn(LD_REND, "Introduction point ntor encryption key is invalid"); - goto err; - } - ip->enc_key_type = HS_DESC_KEY_TYPE_CURVE25519; - } else if (!strcmp(tok->args[0], "legacy")) { - /* "enc-key" SP "legacy" NL key NL */ - if (!tok->key) { - log_warn(LD_REND, "Introduction point legacy encryption key is " - "invalid"); + if (curve25519_public_from_base64(&ip->enc_key, tok->args[1]) < 0) { + log_warn(LD_REND, "Introduction point ntor enc-key is invalid"); goto err; } - ip->enc_key.legacy = crypto_pk_dup_key(tok->key); - ip->enc_key_type = HS_DESC_KEY_TYPE_LEGACY; } else { /* Unknown key type so we can't use that introduction point. */ log_warn(LD_REND, "Introduction point encryption key is unrecognized."); goto err; } - /* "enc-key-certification" NL certificate NL */ - tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY_CERTIFICATION); + /* Exactly once "enc-key-cert" NL certificate NL */ + tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY_CERT); tor_assert(tok->object_body); /* Do the cross certification. */ - switch (ip->enc_key_type) { - case HS_DESC_KEY_TYPE_CURVE25519: - { - if (strcmp(tok->object_type, "ED25519 CERT")) { + if (strcmp(tok->object_type, "ED25519 CERT")) { log_warn(LD_REND, "Introduction point ntor encryption key " "cross-certification has an unknown format."); goto err; - } - if (cert_parse_and_validate(&cross_cert, tok->object_body, - tok->object_size, CERT_TYPE_CROSS_HS_IP_KEYS, - "introduction point enc-key-certification") < 0) { - goto err; - } - break; } - case HS_DESC_KEY_TYPE_LEGACY: - if (strcmp(tok->object_type, "CROSSCERT")) { - log_warn(LD_REND, "Introduction point legacy encryption key " - "cross-certification has an unknown format."); - goto err; - } - if (rsa_ed25519_crosscert_check((const uint8_t *) tok->object_body, - tok->object_size, ip->enc_key.legacy, - &desc->plaintext_data.signing_key_cert->signed_key, - approx_time()-86400)) { - log_warn(LD_REND, "Unable to check cross-certification on the " - "introduction point legacy encryption key."); - goto err; - } - break; - default: - tor_assert(0); - break; + if (cert_parse_and_validate(&ip->enc_key_cert, tok->object_body, + tok->object_size, CERT_TYPE_CROSS_HS_IP_KEYS, + "introduction point enc-key-cert") < 0) { + goto err; + } + if (tor_cert_checksig(ip->enc_key_cert, + &desc->plaintext_data.signing_pubkey, 0) < 0) { + log_warn(LD_REND, "Invalid encryption key signature"); + goto err; } /* It is successfully cross certified. Flag the object. */ ip->cross_certified = 1; + + /* Do we have a "legacy-key" SP key NL ?*/ + tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY); + if (tok) { + if (decode_intro_legacy_key(tok, tokens, ip, desc) < 0) { + goto err; + } + } + + /* Introduction point has been parsed successfully. */ goto done; err: @@ -1271,10 +1736,11 @@ decode_introduction_point(const hs_descriptor_t *desc, const char *start) ip = NULL; done: - tor_cert_free(cross_cert); SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - memarea_drop_all(area); + if (area) { + memarea_drop_all(area); + } return ip; } @@ -1493,8 +1959,8 @@ desc_decode_plaintext_v3(smartlist_t *tokens, /* Copy the encrypted blob to the descriptor object so we can handle it * latter if needed. */ - desc->encrypted_blob = tor_memdup(tok->object_body, tok->object_size); - desc->encrypted_blob_size = tok->object_size; + desc->superencrypted_blob = tor_memdup(tok->object_body, tok->object_size); + desc->superencrypted_blob_size = tok->object_size; /* Extract signature and verify it. */ tok = find_by_keyword(tokens, R3_SIGNATURE); @@ -1528,10 +1994,9 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc, tor_assert(desc); tor_assert(desc_encrypted_out); - /* Decrypt the encrypted data that is located in the plaintext section in - * the descriptor as a blob of bytes. The following functions will use the - * keys found in the same section. */ - message_len = desc_decrypt_data_v3(desc, &message); + /* Decrypt the superencrypted data that is located in the plaintext section + * in the descriptor as a blob of bytes. */ + message_len = desc_decrypt_all(desc, &message); if (!message_len) { log_warn(LD_REND, "Service descriptor decryption failed."); goto err; @@ -1557,7 +2022,7 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc, } /* Authentication type. It's optional but only once. */ - tok = find_opt_by_keyword(tokens, R3_AUTHENTICATION_REQUIRED); + tok = find_opt_by_keyword(tokens, R3_INTRO_AUTH_REQUIRED); if (tok) { if (!decode_auth_type(desc_encrypted_out, tok->args[0])) { log_warn(LD_REND, "Service descriptor authentication type has " @@ -1639,7 +2104,7 @@ hs_desc_decode_encrypted(const hs_descriptor_t *desc, /* Calling this function without an encrypted blob to parse is a code flow * error. The plaintext parsing should never succeed in the first place * without an encrypted section. */ - tor_assert(desc->plaintext_data.encrypted_blob); + tor_assert(desc->plaintext_data.superencrypted_blob); /* Let's make sure we have a supported version as well. By correctly parsing * the plaintext, this should not fail. */ if (BUG(!hs_desc_is_supported_version(version))) { @@ -1891,6 +2356,6 @@ hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data) { tor_assert(data); return (sizeof(*data) + sizeof(*data->signing_key_cert) + - data->encrypted_blob_size); + data->superencrypted_blob_size); } diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h index b520d24471..b8b94792de 100644 --- a/src/or/hs_descriptor.h +++ b/src/or/hs_descriptor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -41,24 +41,11 @@ * the secret IV and MAC key length which is the length of H() output. */ #define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \ CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN -/* We need to pad the plaintext version of the encrypted data section before - * encryption and it has to be a multiple of this value. */ -#define HS_DESC_PLAINTEXT_PADDING_MULTIPLE 128 -/* XXX: Let's make sure this makes sense as an upper limit for the padded - * plaintext section. Then we should enforce it as now only an assert will be - * triggered if we are above it. */ -/* Once padded, this is the maximum length in bytes for the plaintext. */ -#define HS_DESC_PADDED_PLAINTEXT_MAX_LEN 8192 -/* Minimum length in bytes of the encrypted portion of the descriptor. */ -#define HS_DESC_ENCRYPTED_MIN_LEN \ - HS_DESC_ENCRYPTED_SALT_LEN + \ - HS_DESC_PLAINTEXT_PADDING_MULTIPLE + DIGEST256_LEN +/* Pad plaintext of superencrypted data section before encryption so that its + * length is a multiple of this value. */ +#define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000 /* Maximum length in bytes of a full hidden service descriptor. */ #define HS_DESC_MAX_LEN 50000 /* 50kb max size */ -/* The minimum amount of fields a descriptor should contain. The parsing of - * the fields are version specific so the only required field, as a generic - * view of a descriptor, is 1 that is the version field. */ -#define HS_DESC_PLAINTEXT_MIN_FIELDS 1 /* Key length for the descriptor symmetric encryption. As specified in the * protocol, we use AES-256 for the encrypted section of the descriptor. The @@ -68,16 +55,9 @@ /* Type of authentication in the descriptor. */ typedef enum { - HS_DESC_AUTH_PASSWORD = 1, - HS_DESC_AUTH_ED25519 = 2, + HS_DESC_AUTH_ED25519 = 1 } hs_desc_auth_type_t; -/* Type of encryption key in the descriptor. */ -typedef enum { - HS_DESC_KEY_TYPE_LEGACY = 1, - HS_DESC_KEY_TYPE_CURVE25519 = 2, -} hs_desc_key_type_t; - /* Link specifier object that contains information on how to extend to the * relay that is the address, port and handshake type. */ typedef struct hs_desc_link_specifier_t { @@ -105,18 +85,29 @@ typedef struct hs_desc_intro_point_t { * the blinded key and in turn signs it. */ tor_cert_t *auth_key_cert; - /* Encryption key type so we know which one to use in the union below. */ - hs_desc_key_type_t enc_key_type; - - /* Keys are mutually exclusive thus the union. */ - union { - /* Encryption key used to encrypt request to hidden service. */ - curve25519_keypair_t curve25519; - - /* Backward compat: RSA 1024 encryption key for legacy purposes. - * Mutually exclusive with enc_key. */ - crypto_pk_t *legacy; - } enc_key; + /* Encryption key for the "ntor" type. */ + curve25519_public_key_t enc_key; + + /* Certificate cross certifying the descriptor signing key by the encryption + * curve25519 key. This certificate contains the signing key and is of type + * CERT_TYPE_CROSS_HS_IP_KEYS [0B]. */ + tor_cert_t *enc_key_cert; + + /* (Optional): If this introduction point is a legacy one that is version <= + * 0.2.9.x (HSIntro=3), we use this extra key for the intro point to be able + * to relay the cells to the service correctly. */ + struct { + /* RSA public key. */ + crypto_pk_t *key; + + /* Cross certified cert with the descriptor signing key (RSA->Ed). Because + * of the cross certification API, we need to keep the certificate binary + * blob and its length in order to properly encode it after. */ + struct { + uint8_t *encoded; + size_t len; + } cert; + } legacy; /* True iff the introduction point has passed the cross certification. Upon * decoding an intro point, this must be true. */ @@ -132,7 +123,7 @@ typedef struct hs_desc_encrypted_data_t { /* A list of authentication types that a client must at least support one * in order to contact the service. Contains NULL terminated strings. */ - smartlist_t *auth_types; + smartlist_t *intro_auth_types; /* Is this descriptor a single onion service? */ unsigned int single_onion_service : 1; @@ -167,11 +158,11 @@ typedef struct hs_desc_plaintext_data_t { * has changed. Spec specifies this as a 8 bytes positive integer. */ uint64_t revision_counter; - /* Decoding only: The base64-decoded encrypted blob from the descriptor */ - uint8_t *encrypted_blob; + /* Decoding only: The b64-decoded superencrypted blob from the descriptor */ + uint8_t *superencrypted_blob; - /* Decoding only: Size of the encrypted_blob */ - size_t encrypted_blob_size; + /* Decoding only: Size of the superencrypted_blob */ + size_t superencrypted_blob_size; } hs_desc_plaintext_data_t; /* Service descriptor in its decoded form. */ @@ -242,6 +233,8 @@ STATIC int desc_sig_is_valid(const char *b64_sig, const ed25519_public_key_t *signing_pubkey, const char *encoded_desc, size_t encoded_len); STATIC void desc_intro_point_free(hs_desc_intro_point_t *ip); +STATIC size_t decode_superencrypted(const char *message, size_t message_len, + uint8_t **encrypted_out); #endif /* HS_DESCRIPTOR_PRIVATE */ #endif /* TOR_HS_DESCRIPTOR_H */ diff --git a/src/or/hs_intropoint.c b/src/or/hs_intropoint.c index db4ba7982a..2abbfcd6c3 100644 --- a/src/or/hs_intropoint.c +++ b/src/or/hs_intropoint.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -43,16 +43,16 @@ get_auth_key_from_cell(ed25519_public_key_t *auth_key_out, switch (cell_type) { case RELAY_COMMAND_ESTABLISH_INTRO: { - const hs_cell_establish_intro_t *c_cell = cell; - key_array = hs_cell_establish_intro_getconstarray_auth_key(c_cell); - auth_key_len = hs_cell_establish_intro_getlen_auth_key(c_cell); + const trn_cell_establish_intro_t *c_cell = cell; + key_array = trn_cell_establish_intro_getconstarray_auth_key(c_cell); + auth_key_len = trn_cell_establish_intro_getlen_auth_key(c_cell); break; } case RELAY_COMMAND_INTRODUCE1: { - const hs_cell_introduce1_t *c_cell = cell; - key_array = hs_cell_introduce1_getconstarray_auth_key(cell); - auth_key_len = hs_cell_introduce1_getlen_auth_key(c_cell); + const trn_cell_introduce1_t *c_cell = cell; + key_array = trn_cell_introduce1_getconstarray_auth_key(cell); + auth_key_len = trn_cell_introduce1_getlen_auth_key(c_cell); break; } default: @@ -68,22 +68,22 @@ get_auth_key_from_cell(ed25519_public_key_t *auth_key_out, /** We received an ESTABLISH_INTRO <b>cell</b>. Verify its signature and MAC, * given <b>circuit_key_material</b>. Return 0 on success else -1 on error. */ STATIC int -verify_establish_intro_cell(const hs_cell_establish_intro_t *cell, +verify_establish_intro_cell(const trn_cell_establish_intro_t *cell, const uint8_t *circuit_key_material, size_t circuit_key_material_len) { /* We only reach this function if the first byte of the cell is 0x02 which - * means that auth_key_type is AUTH_KEY_ED25519, hence this check should + * means that auth_key_type is of ed25519 type, hence this check should * always pass. See hs_intro_received_establish_intro(). */ - if (BUG(cell->auth_key_type != AUTH_KEY_ED25519)) { + if (BUG(cell->auth_key_type != HS_INTRO_AUTH_KEY_TYPE_ED25519)) { return -1; } /* Make sure the auth key length is of the right size for this type. For * EXTRA safety, we check both the size of the array and the length which * must be the same. Safety first!*/ - if (hs_cell_establish_intro_getlen_auth_key(cell) != ED25519_PUBKEY_LEN || - hs_cell_establish_intro_get_auth_key_len(cell) != ED25519_PUBKEY_LEN) { + if (trn_cell_establish_intro_getlen_auth_key(cell) != ED25519_PUBKEY_LEN || + trn_cell_establish_intro_get_auth_key_len(cell) != ED25519_PUBKEY_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "ESTABLISH_INTRO auth key length is invalid"); return -1; @@ -94,13 +94,14 @@ verify_establish_intro_cell(const hs_cell_establish_intro_t *cell, /* Verify the sig */ { ed25519_signature_t sig_struct; - const uint8_t *sig_array = hs_cell_establish_intro_getconstarray_sig(cell); + const uint8_t *sig_array = + trn_cell_establish_intro_getconstarray_sig(cell); /* Make sure the signature length is of the right size. For EXTRA safety, * we check both the size of the array and the length which must be the * same. Safety first!*/ - if (hs_cell_establish_intro_getlen_sig(cell) != sizeof(sig_struct.sig) || - hs_cell_establish_intro_get_sig_len(cell) != sizeof(sig_struct.sig)) { + if (trn_cell_establish_intro_getlen_sig(cell) != sizeof(sig_struct.sig) || + trn_cell_establish_intro_get_sig_len(cell) != sizeof(sig_struct.sig)) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "ESTABLISH_INTRO sig len is invalid"); return -1; @@ -147,21 +148,21 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ)) int ret; uint8_t *encoded_cell = NULL; ssize_t encoded_len, result_len; - hs_cell_intro_established_t *cell; - cell_extension_t *ext; + trn_cell_intro_established_t *cell; + trn_cell_extension_t *ext; tor_assert(circ); /* Build the cell payload. */ - cell = hs_cell_intro_established_new(); - ext = cell_extension_new(); - cell_extension_set_num(ext, 0); - hs_cell_intro_established_set_extensions(cell, ext); + cell = trn_cell_intro_established_new(); + ext = trn_cell_extension_new(); + trn_cell_extension_set_num(ext, 0); + trn_cell_intro_established_set_extensions(cell, ext); /* Encode the cell to binary format. */ - encoded_len = hs_cell_intro_established_encoded_len(cell); + encoded_len = trn_cell_intro_established_encoded_len(cell); tor_assert(encoded_len > 0); encoded_cell = tor_malloc_zero(encoded_len); - result_len = hs_cell_intro_established_encode(encoded_cell, encoded_len, + result_len = trn_cell_intro_established_encode(encoded_cell, encoded_len, cell); tor_assert(encoded_len == result_len); @@ -170,7 +171,7 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ)) (char *) encoded_cell, encoded_len, NULL); /* On failure, the above function will close the circuit. */ - hs_cell_intro_established_free(cell); + trn_cell_intro_established_free(cell); tor_free(encoded_cell); return ret; } @@ -180,7 +181,7 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ)) * establish an intro point. */ static int handle_verified_establish_intro_cell(or_circuit_t *circ, - const hs_cell_establish_intro_t *parsed_cell) + const trn_cell_establish_intro_t *parsed_cell) { /* Get the auth key of this intro point */ ed25519_public_key_t auth_key; @@ -195,7 +196,7 @@ handle_verified_establish_intro_cell(or_circuit_t *circ, } /* Associate intro point auth key with this circuit. */ - hs_circuitmap_register_intro_circ_v3(circ, &auth_key); + hs_circuitmap_register_intro_circ_v3_relay_side(circ, &auth_key); /* Repurpose this circuit into an intro circuit. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT); @@ -210,7 +211,7 @@ handle_establish_intro(or_circuit_t *circ, const uint8_t *request, size_t request_len) { int cell_ok, retval = -1; - hs_cell_establish_intro_t *parsed_cell = NULL; + trn_cell_establish_intro_t *parsed_cell = NULL; tor_assert(circ); tor_assert(request); @@ -224,7 +225,7 @@ handle_establish_intro(or_circuit_t *circ, const uint8_t *request, } /* Parse the cell */ - ssize_t parsing_result = hs_cell_establish_intro_parse(&parsed_cell, + ssize_t parsing_result = trn_cell_establish_intro_parse(&parsed_cell, request, request_len); if (parsing_result < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -259,7 +260,7 @@ handle_establish_intro(or_circuit_t *circ, const uint8_t *request, circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL); done: - hs_cell_establish_intro_free(parsed_cell); + trn_cell_establish_intro_free(parsed_cell); return retval; } @@ -339,28 +340,28 @@ send_introduce_ack_cell(or_circuit_t *circ, hs_intro_ack_status_t status) int ret = -1; uint8_t *encoded_cell = NULL; ssize_t encoded_len, result_len; - hs_cell_introduce_ack_t *cell; - cell_extension_t *ext; + trn_cell_introduce_ack_t *cell; + trn_cell_extension_t *ext; tor_assert(circ); /* Setup the INTRODUCE_ACK cell. We have no extensions so the N_EXTENSIONS * field is set to 0 by default with a new object. */ - cell = hs_cell_introduce_ack_new(); - ret = hs_cell_introduce_ack_set_status(cell, status); + cell = trn_cell_introduce_ack_new(); + ret = trn_cell_introduce_ack_set_status(cell, status); /* We have no cell extensions in an INTRODUCE_ACK cell. */ - ext = cell_extension_new(); - cell_extension_set_num(ext, 0); - hs_cell_introduce_ack_set_extensions(cell, ext); + ext = trn_cell_extension_new(); + trn_cell_extension_set_num(ext, 0); + trn_cell_introduce_ack_set_extensions(cell, ext); /* A wrong status is a very bad code flow error as this value is controlled * by the code in this file and not an external input. This means we use a * code that is not known by the trunnel ABI. */ tor_assert(ret == 0); /* Encode the payload. We should never fail to get the encoded length. */ - encoded_len = hs_cell_introduce_ack_encoded_len(cell); + encoded_len = trn_cell_introduce_ack_encoded_len(cell); tor_assert(encoded_len > 0); encoded_cell = tor_malloc_zero(encoded_len); - result_len = hs_cell_introduce_ack_encode(encoded_cell, encoded_len, cell); + result_len = trn_cell_introduce_ack_encode(encoded_cell, encoded_len, cell); tor_assert(encoded_len == result_len); ret = relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ), @@ -368,7 +369,7 @@ send_introduce_ack_cell(or_circuit_t *circ, hs_intro_ack_status_t status) (char *) encoded_cell, encoded_len, NULL); /* On failure, the above function will close the circuit. */ - hs_cell_introduce_ack_free(cell); + trn_cell_introduce_ack_free(cell); tor_free(encoded_cell); return ret; } @@ -376,7 +377,7 @@ send_introduce_ack_cell(or_circuit_t *circ, hs_intro_ack_status_t status) /* Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a * negative value for an invalid cell that should be NACKed. */ STATIC int -validate_introduce1_parsed_cell(const hs_cell_introduce1_t *cell) +validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell) { size_t legacy_key_id_len; const uint8_t *legacy_key_id; @@ -385,29 +386,29 @@ validate_introduce1_parsed_cell(const hs_cell_introduce1_t *cell) /* This code path SHOULD NEVER be reached if the cell is a legacy type so * safety net here. The legacy ID must be zeroes in this case. */ - legacy_key_id_len = hs_cell_introduce1_getlen_legacy_key_id(cell); - legacy_key_id = hs_cell_introduce1_getconstarray_legacy_key_id(cell); + legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell); + legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell); if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) { goto invalid; } /* The auth key of an INTRODUCE1 should be of type ed25519 thus leading to a * known fixed length as well. */ - if (hs_cell_introduce1_get_auth_key_type(cell) != + if (trn_cell_introduce1_get_auth_key_type(cell) != HS_INTRO_AUTH_KEY_TYPE_ED25519) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting invalid INTRODUCE1 cell auth key type. " "Responding with NACK."); goto invalid; } - if (hs_cell_introduce1_get_auth_key_len(cell) != ED25519_PUBKEY_LEN || - hs_cell_introduce1_getlen_auth_key(cell) != ED25519_PUBKEY_LEN) { + if (trn_cell_introduce1_get_auth_key_len(cell) != ED25519_PUBKEY_LEN || + trn_cell_introduce1_getlen_auth_key(cell) != ED25519_PUBKEY_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting invalid INTRODUCE1 cell auth key length. " "Responding with NACK."); goto invalid; } - if (hs_cell_introduce1_getlen_encrypted(cell) == 0) { + if (trn_cell_introduce1_getlen_encrypted(cell) == 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting invalid INTRODUCE1 cell encrypted length. " "Responding with NACK."); @@ -430,7 +431,7 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request, { int ret = -1; or_circuit_t *service_circ; - hs_cell_introduce1_t *parsed_cell; + trn_cell_introduce1_t *parsed_cell; hs_intro_ack_status_t status = HS_INTRO_ACK_STATUS_SUCCESS; tor_assert(client_circ); @@ -439,7 +440,7 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request, /* Parse cell. Note that we can only parse the non encrypted section for * which we'll use the authentication key to find the service introduction * circuit and relay the cell on it. */ - ssize_t cell_size = hs_cell_introduce1_parse(&parsed_cell, request, + ssize_t cell_size = trn_cell_introduce1_parse(&parsed_cell, request, request_len); if (cell_size < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -461,7 +462,7 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request, { ed25519_public_key_t auth_key; get_auth_key_from_cell(&auth_key, RELAY_COMMAND_INTRODUCE1, parsed_cell); - service_circ = hs_circuitmap_get_intro_circ_v3(&auth_key); + service_circ = hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key); if (service_circ == NULL) { char b64_key[ED25519_BASE64_LEN + 1]; ed25519_public_to_base64(b64_key, &auth_key); @@ -505,7 +506,7 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request, circuit_mark_for_close(TO_CIRCUIT(client_circ), END_CIRC_REASON_INTERNAL); } done: - hs_cell_introduce1_free(parsed_cell); + trn_cell_introduce1_free(parsed_cell); return ret; } diff --git a/src/or/hs_intropoint.h b/src/or/hs_intropoint.h index e6024a858f..163ed810e7 100644 --- a/src/or/hs_intropoint.h +++ b/src/or/hs_intropoint.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -41,7 +41,7 @@ int hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ); #include "hs/cell_introduce1.h" STATIC int -verify_establish_intro_cell(const hs_cell_establish_intro_t *out, +verify_establish_intro_cell(const trn_cell_establish_intro_t *out, const uint8_t *circuit_key_material, size_t circuit_key_material_len); @@ -52,7 +52,7 @@ get_auth_key_from_cell(ed25519_public_key_t *auth_key_out, STATIC int introduce1_cell_is_legacy(const uint8_t *request); STATIC int handle_introduce1(or_circuit_t *client_circ, const uint8_t *request, size_t request_len); -STATIC int validate_introduce1_parsed_cell(const hs_cell_introduce1_t *cell); +STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell); STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ); #endif /* HS_INTROPOINT_PRIVATE */ diff --git a/src/or/hs_ntor.c b/src/or/hs_ntor.c new file mode 100644 index 0000000000..119899817e --- /dev/null +++ b/src/or/hs_ntor.c @@ -0,0 +1,626 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** \file hs_ntor.c + * \brief Implements the ntor variant used in Tor hidden services. + * + * \details + * This module handles the variant of the ntor handshake that is documented in + * section [NTOR-WITH-EXTRA-DATA] of rend-spec-ng.txt . + * + * The functions in this file provide an API that should be used when sending + * or receiving INTRODUCE1/RENDEZVOUS1 cells to generate the various key + * material required to create and handle those cells. + * + * In the case of INTRODUCE1 it provides encryption and MAC keys to + * encode/decode the encrypted blob (see hs_ntor_intro_cell_keys_t). The + * relevant pub functions are hs_ntor_{client,service}_get_introduce1_keys(). + * + * In the case of RENDEZVOUS1 it calculates the MAC required to authenticate + * the cell, and also provides the key seed that is used to derive the crypto + * material for rendezvous encryption (see hs_ntor_rend_cell_keys_t). The + * relevant pub functions are hs_ntor_{client,service}_get_rendezvous1_keys(). + * It also provides a function (hs_ntor_circuit_key_expansion()) that does the + * rendezvous key expansion to setup end-to-end rend circuit keys. + */ + +#include "or.h" +#include "hs_ntor.h" + +/* String constants used by the ntor HS protocol */ +#define PROTOID "tor-hs-ntor-curve25519-sha3-256-1" +#define PROTOID_LEN (sizeof(PROTOID) - 1) +#define SERVER_STR "Server" +#define SERVER_STR_LEN (sizeof(SERVER_STR) - 1) + +/* Protocol-specific tweaks to our crypto inputs */ +#define T_HSENC PROTOID ":hs_key_extract" +#define T_HSENC_LEN (sizeof(T_HSENC) - 1) +#define T_HSVERIFY PROTOID ":hs_verify" +#define T_HSMAC PROTOID ":hs_mac" +#define M_HSEXPAND PROTOID ":hs_key_expand" +#define M_HSEXPAND_LEN (sizeof(M_HSEXPAND) - 1) + +/************************* Helper functions: *******************************/ + +/** Helper macro: copy <b>len</b> bytes from <b>inp</b> to <b>ptr</b> and + *advance <b>ptr</b> by the number of bytes copied. Stolen from onion_ntor.c */ +#define APPEND(ptr, inp, len) \ + STMT_BEGIN { \ + memcpy(ptr, (inp), (len)); \ + ptr += len; \ + } STMT_END + +/* Length of EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID */ +#define REND_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN * 2 + \ + ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN) +/* Length of auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" */ +#define REND_AUTH_INPUT_LEN (DIGEST256_LEN + ED25519_PUBKEY_LEN + \ + CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN + SERVER_STR_LEN) + +/** Helper function: Compute the last part of the HS ntor handshake which + * derives key material necessary to create and handle RENDEZVOUS1 + * cells. Function used by both client and service. The actual calculations is + * as follows: + * + * NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc) + * verify = MAC(rend_secret_hs_input, t_hsverify) + * auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" + * auth_input_mac = MAC(auth_input, t_hsmac) + * + * where in the above, AUTH_KEY is <b>intro_auth_pubkey</b>, B is + * <b>intro_enc_pubkey</b>, Y is <b>service_ephemeral_rend_pubkey</b>, and X + * is <b>client_ephemeral_enc_pubkey</b>. The provided + * <b>rend_secret_hs_input</b> is of size REND_SECRET_HS_INPUT_LEN. + * + * The final results of NTOR_KEY_SEED and auth_input_mac are placed in + * <b>hs_ntor_rend_cell_keys_out</b>. Return 0 if everything went fine. */ +static int +get_rendezvous1_key_material(const uint8_t *rend_secret_hs_input, + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_public_key_t *intro_enc_pubkey, + const curve25519_public_key_t *service_ephemeral_rend_pubkey, + const curve25519_public_key_t *client_ephemeral_enc_pubkey, + hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out) +{ + int bad = 0; + uint8_t ntor_key_seed[DIGEST256_LEN]; + uint8_t ntor_verify[DIGEST256_LEN]; + uint8_t rend_auth_input[REND_AUTH_INPUT_LEN]; + uint8_t rend_cell_auth[DIGEST256_LEN]; + uint8_t *ptr; + + /* Let's build NTOR_KEY_SEED */ + crypto_mac_sha3_256(ntor_key_seed, sizeof(ntor_key_seed), + rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN, + (const uint8_t *)T_HSENC, strlen(T_HSENC)); + bad |= safe_mem_is_zero(ntor_key_seed, DIGEST256_LEN); + + /* Let's build ntor_verify */ + crypto_mac_sha3_256(ntor_verify, sizeof(ntor_verify), + rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN, + (const uint8_t *)T_HSVERIFY, strlen(T_HSVERIFY)); + bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN); + + /* Let's build auth_input: */ + ptr = rend_auth_input; + /* Append ntor_verify */ + APPEND(ptr, ntor_verify, sizeof(ntor_verify)); + /* Append AUTH_KEY */ + APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN); + /* Append B */ + APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append Y */ + APPEND(ptr, + service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append X */ + APPEND(ptr, + client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append PROTOID */ + APPEND(ptr, PROTOID, strlen(PROTOID)); + /* Append "Server" */ + APPEND(ptr, SERVER_STR, strlen(SERVER_STR)); + tor_assert(ptr == rend_auth_input + sizeof(rend_auth_input)); + + /* Let's build auth_input_mac that goes in RENDEZVOUS1 cell */ + crypto_mac_sha3_256(rend_cell_auth, sizeof(rend_cell_auth), + rend_auth_input, sizeof(rend_auth_input), + (const uint8_t *)T_HSMAC, strlen(T_HSMAC)); + bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN); + + { /* Get the computed RENDEZVOUS1 material! */ + memcpy(&hs_ntor_rend_cell_keys_out->rend_cell_auth_mac, + rend_cell_auth, DIGEST256_LEN); + memcpy(&hs_ntor_rend_cell_keys_out->ntor_key_seed, + ntor_key_seed, DIGEST256_LEN); + } + + memwipe(rend_cell_auth, 0, sizeof(rend_cell_auth)); + memwipe(rend_auth_input, 0, sizeof(rend_auth_input)); + memwipe(ntor_key_seed, 0, sizeof(ntor_key_seed)); + + return bad; +} + +/** Length of secret_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID */ +#define INTRO_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN +ED25519_PUBKEY_LEN +\ + CURVE25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN + PROTOID_LEN) +/* Length of info = m_hsexpand | subcredential */ +#define INFO_BLOB_LEN (M_HSEXPAND_LEN + DIGEST256_LEN) +/* Length of KDF input = intro_secret_hs_input | t_hsenc | info */ +#define KDF_INPUT_LEN (INTRO_SECRET_HS_INPUT_LEN + T_HSENC_LEN + INFO_BLOB_LEN) + +/** Helper function: Compute the part of the HS ntor handshake that generates + * key material for creating and handling INTRODUCE1 cells. Function used + * by both client and service. Specifically, calculate the following: + * + * info = m_hsexpand | subcredential + * hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN) + * ENC_KEY = hs_keys[0:S_KEY_LEN] + * MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN] + * + * where intro_secret_hs_input is <b>secret_input</b> (of size + * INTRO_SECRET_HS_INPUT_LEN), and <b>subcredential</b> is of size + * DIGEST256_LEN. + * + * If everything went well, fill <b>hs_ntor_intro_cell_keys_out</b> with the + * necessary key material, and return 0. */ +static void +get_introduce1_key_material(const uint8_t *secret_input, + const uint8_t *subcredential, + hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out) +{ + uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN]; + uint8_t info_blob[INFO_BLOB_LEN]; + uint8_t kdf_input[KDF_INPUT_LEN]; + crypto_xof_t *xof; + uint8_t *ptr; + + /* Let's build info */ + ptr = info_blob; + APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND)); + APPEND(ptr, subcredential, DIGEST256_LEN); + tor_assert(ptr == info_blob + sizeof(info_blob)); + + /* Let's build the input to the KDF */ + ptr = kdf_input; + APPEND(ptr, secret_input, INTRO_SECRET_HS_INPUT_LEN); + APPEND(ptr, T_HSENC, strlen(T_HSENC)); + APPEND(ptr, info_blob, sizeof(info_blob)); + tor_assert(ptr == kdf_input + sizeof(kdf_input)); + + /* Now we need to run kdf_input over SHAKE-256 */ + xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input)); + crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ; + crypto_xof_free(xof); + + { /* Get the keys */ + memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN); + memcpy(&hs_ntor_intro_cell_keys_out->mac_key, + keystream+CIPHER256_KEY_LEN, DIGEST256_LEN); + } + + memwipe(keystream, 0, sizeof(keystream)); + memwipe(kdf_input, 0, sizeof(kdf_input)); +} + +/** Helper function: Calculate the 'intro_secret_hs_input' element used by the + * HS ntor handshake and place it in <b>secret_input_out</b>. This function is + * used by both client and service code. + * + * For the client-side it looks like this: + * + * intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID + * + * whereas for the service-side it looks like this: + * + * intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID + * + * In this function, <b>dh_result</b> carries the EXP() result (and has size + * CURVE25519_OUTPUT_LEN) <b>intro_auth_pubkey</b> is AUTH_KEY, + * <b>client_ephemeral_enc_pubkey</b> is X, and <b>intro_enc_pubkey</b> is B. + */ +static void +get_intro_secret_hs_input(const uint8_t *dh_result, + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_public_key_t *client_ephemeral_enc_pubkey, + const curve25519_public_key_t *intro_enc_pubkey, + uint8_t *secret_input_out) +{ + uint8_t *ptr; + + /* Append EXP() */ + ptr = secret_input_out; + APPEND(ptr, dh_result, CURVE25519_OUTPUT_LEN); + /* Append AUTH_KEY */ + APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN); + /* Append X */ + APPEND(ptr, client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append B */ + APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append PROTOID */ + APPEND(ptr, PROTOID, strlen(PROTOID)); + tor_assert(ptr == secret_input_out + INTRO_SECRET_HS_INPUT_LEN); +} + +/** Calculate the 'rend_secret_hs_input' element used by the HS ntor handshake + * and place it in <b>rend_secret_hs_input_out</b>. This function is used by + * both client and service code. + * + * The computation on the client side is: + * rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID + * whereas on the service side it is: + * rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID + * + * where: + * <b>dh_result1</b> and <b>dh_result2</b> carry the two EXP() results (of size + * CURVE25519_OUTPUT_LEN) + * <b>intro_auth_pubkey</b> is AUTH_KEY, + * <b>intro_enc_pubkey</b> is B, + * <b>client_ephemeral_enc_pubkey</b> is X, and + * <b>service_ephemeral_rend_pubkey</b> is Y. + */ +static void +get_rend_secret_hs_input(const uint8_t *dh_result1, const uint8_t *dh_result2, + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_public_key_t *intro_enc_pubkey, + const curve25519_public_key_t *client_ephemeral_enc_pubkey, + const curve25519_public_key_t *service_ephemeral_rend_pubkey, + uint8_t *rend_secret_hs_input_out) +{ + uint8_t *ptr; + + ptr = rend_secret_hs_input_out; + /* Append the first EXP() */ + APPEND(ptr, dh_result1, CURVE25519_OUTPUT_LEN); + /* Append the other EXP() */ + APPEND(ptr, dh_result2, CURVE25519_OUTPUT_LEN); + /* Append AUTH_KEY */ + APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN); + /* Append B */ + APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append X */ + APPEND(ptr, + client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append Y */ + APPEND(ptr, + service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN); + /* Append PROTOID */ + APPEND(ptr, PROTOID, strlen(PROTOID)); + tor_assert(ptr == rend_secret_hs_input_out + REND_SECRET_HS_INPUT_LEN); +} + +/************************* Public functions: *******************************/ + +/* Public function: Do the appropriate ntor calculations and derive the keys + * needed to encrypt and authenticate INTRODUCE1 cells. Return 0 and place the + * final key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went + * well, otherwise return -1; + * + * The relevant calculations are as follows: + * + * intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID + * info = m_hsexpand | subcredential + * hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN) + * ENC_KEY = hs_keys[0:S_KEY_LEN] + * MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN] + * + * where: + * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor), + * <b>intro_enc_pubkey</b> is B (also found in HS descriptor), + * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X) + * <b>subcredential</b> is the hidden service subcredential (of size + * DIGEST256_LEN). */ +int +hs_ntor_client_get_introduce1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_public_key_t *intro_enc_pubkey, + const curve25519_keypair_t *client_ephemeral_enc_keypair, + const uint8_t *subcredential, + hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out) +{ + int bad = 0; + uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN]; + uint8_t dh_result[CURVE25519_OUTPUT_LEN]; + + tor_assert(intro_auth_pubkey); + tor_assert(intro_enc_pubkey); + tor_assert(client_ephemeral_enc_keypair); + tor_assert(subcredential); + tor_assert(hs_ntor_intro_cell_keys_out); + + /* Calculate EXP(B,x) */ + curve25519_handshake(dh_result, + &client_ephemeral_enc_keypair->seckey, + intro_enc_pubkey); + bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN); + + /* Get intro_secret_hs_input */ + get_intro_secret_hs_input(dh_result, intro_auth_pubkey, + &client_ephemeral_enc_keypair->pubkey, + intro_enc_pubkey, secret_input); + bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN); + + /* Get ENC_KEY and MAC_KEY! */ + get_introduce1_key_material(secret_input, subcredential, + hs_ntor_intro_cell_keys_out); + + /* Cleanup */ + memwipe(secret_input, 0, sizeof(secret_input)); + if (bad) { + memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t)); + } + + return bad ? -1 : 0; +} + +/* Public function: Do the appropriate ntor calculations and derive the keys + * needed to verify RENDEZVOUS1 cells and encrypt further rendezvous + * traffic. Return 0 and place the final key material in + * <b>hs_ntor_rend_cell_keys_out</b> if everything went well, else return -1. + * + * The relevant calculations are as follows: + * + * rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID + * NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc) + * verify = MAC(rend_secret_hs_input, t_hsverify) + * auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" + * auth_input_mac = MAC(auth_input, t_hsmac) + * + * where: + * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor), + * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X) + * <b>intro_enc_pubkey</b> is B (also found in HS descriptor), + * <b>service_ephemeral_rend_pubkey</b> is Y (SERVER_PK in RENDEZVOUS1 cell) */ +int +hs_ntor_client_get_rendezvous1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_keypair_t *client_ephemeral_enc_keypair, + const curve25519_public_key_t *intro_enc_pubkey, + const curve25519_public_key_t *service_ephemeral_rend_pubkey, + hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out) +{ + int bad = 0; + uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN]; + uint8_t dh_result1[CURVE25519_OUTPUT_LEN]; + uint8_t dh_result2[CURVE25519_OUTPUT_LEN]; + + tor_assert(intro_auth_pubkey); + tor_assert(client_ephemeral_enc_keypair); + tor_assert(intro_enc_pubkey); + tor_assert(service_ephemeral_rend_pubkey); + tor_assert(hs_ntor_rend_cell_keys_out); + + /* Compute EXP(Y, x) */ + curve25519_handshake(dh_result1, + &client_ephemeral_enc_keypair->seckey, + service_ephemeral_rend_pubkey); + bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN); + + /* Compute EXP(B, x) */ + curve25519_handshake(dh_result2, + &client_ephemeral_enc_keypair->seckey, + intro_enc_pubkey); + bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN); + + /* Get rend_secret_hs_input */ + get_rend_secret_hs_input(dh_result1, dh_result2, + intro_auth_pubkey, intro_enc_pubkey, + &client_ephemeral_enc_keypair->pubkey, + service_ephemeral_rend_pubkey, + rend_secret_hs_input); + + /* Get NTOR_KEY_SEED and the auth_input MAC */ + bad |= get_rendezvous1_key_material(rend_secret_hs_input, + intro_auth_pubkey, + intro_enc_pubkey, + service_ephemeral_rend_pubkey, + &client_ephemeral_enc_keypair->pubkey, + hs_ntor_rend_cell_keys_out); + + memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input)); + if (bad) { + memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t)); + } + + return bad ? -1 : 0; +} + +/* Public function: Do the appropriate ntor calculations and derive the keys + * needed to decrypt and verify INTRODUCE1 cells. Return 0 and place the final + * key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went well, + * otherwise return -1; + * + * The relevant calculations are as follows: + * + * intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID + * info = m_hsexpand | subcredential + * hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN) + * HS_DEC_KEY = hs_keys[0:S_KEY_LEN] + * HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN] + * + * where: + * <b>intro_auth_pubkey</b> is AUTH_KEY (introduction point auth key), + * <b>intro_enc_keypair</b> is (b,B) (introduction point encryption keypair), + * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell), + * <b>subcredential</b> is the HS subcredential (of size DIGEST256_LEN) */ +int +hs_ntor_service_get_introduce1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_keypair_t *intro_enc_keypair, + const curve25519_public_key_t *client_ephemeral_enc_pubkey, + const uint8_t *subcredential, + hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out) +{ + int bad = 0; + uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN]; + uint8_t dh_result[CURVE25519_OUTPUT_LEN]; + + tor_assert(intro_auth_pubkey); + tor_assert(intro_enc_keypair); + tor_assert(client_ephemeral_enc_pubkey); + tor_assert(subcredential); + tor_assert(hs_ntor_intro_cell_keys_out); + + /* Compute EXP(X, b) */ + curve25519_handshake(dh_result, + &intro_enc_keypair->seckey, + client_ephemeral_enc_pubkey); + bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN); + + /* Get intro_secret_hs_input */ + get_intro_secret_hs_input(dh_result, intro_auth_pubkey, + client_ephemeral_enc_pubkey, + &intro_enc_keypair->pubkey, + secret_input); + bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN); + + /* Get ENC_KEY and MAC_KEY! */ + get_introduce1_key_material(secret_input, subcredential, + hs_ntor_intro_cell_keys_out); + + memwipe(secret_input, 0, sizeof(secret_input)); + if (bad) { + memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t)); + } + + return bad ? -1 : 0; +} + +/* Public function: Do the appropriate ntor calculations and derive the keys + * needed to create and authenticate RENDEZVOUS1 cells. Return 0 and place the + * final key material in <b>hs_ntor_rend_cell_keys_out</b> if all went fine, + * return -1 if error happened. + * + * The relevant calculations are as follows: + * + * rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID + * NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc) + * verify = MAC(rend_secret_hs_input, t_hsverify) + * auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" + * auth_input_mac = MAC(auth_input, t_hsmac) + * + * where: + * <b>intro_auth_pubkey</b> is AUTH_KEY (intro point auth key), + * <b>intro_enc_keypair</b> is (b,B) (intro point enc keypair) + * <b>service_ephemeral_rend_keypair</b> is a fresh (y,Y) keypair + * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell) */ +int +hs_ntor_service_get_rendezvous1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_keypair_t *intro_enc_keypair, + const curve25519_keypair_t *service_ephemeral_rend_keypair, + const curve25519_public_key_t *client_ephemeral_enc_pubkey, + hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out) +{ + int bad = 0; + uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN]; + uint8_t dh_result1[CURVE25519_OUTPUT_LEN]; + uint8_t dh_result2[CURVE25519_OUTPUT_LEN]; + + tor_assert(intro_auth_pubkey); + tor_assert(intro_enc_keypair); + tor_assert(service_ephemeral_rend_keypair); + tor_assert(client_ephemeral_enc_pubkey); + tor_assert(hs_ntor_rend_cell_keys_out); + + /* Compute EXP(X, y) */ + curve25519_handshake(dh_result1, + &service_ephemeral_rend_keypair->seckey, + client_ephemeral_enc_pubkey); + bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN); + + /* Compute EXP(X, b) */ + curve25519_handshake(dh_result2, + &intro_enc_keypair->seckey, + client_ephemeral_enc_pubkey); + bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN); + + /* Get rend_secret_hs_input */ + get_rend_secret_hs_input(dh_result1, dh_result2, + intro_auth_pubkey, + &intro_enc_keypair->pubkey, + client_ephemeral_enc_pubkey, + &service_ephemeral_rend_keypair->pubkey, + rend_secret_hs_input); + + /* Get NTOR_KEY_SEED and AUTH_INPUT_MAC! */ + bad |= get_rendezvous1_key_material(rend_secret_hs_input, + intro_auth_pubkey, + &intro_enc_keypair->pubkey, + &service_ephemeral_rend_keypair->pubkey, + client_ephemeral_enc_pubkey, + hs_ntor_rend_cell_keys_out); + + memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input)); + if (bad) { + memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t)); + } + + return bad ? -1 : 0; +} + +/** Given a received RENDEZVOUS2 MAC in <b>mac</b> (of length DIGEST256_LEN), + * and the RENDEZVOUS1 key material in <b>hs_ntor_rend_cell_keys</b>, return 1 + * if the MAC is good, otherwise return 0. */ +int +hs_ntor_client_rendezvous2_mac_is_good( + const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys, + const uint8_t *rcvd_mac) +{ + tor_assert(rcvd_mac); + tor_assert(hs_ntor_rend_cell_keys); + + return tor_memeq(hs_ntor_rend_cell_keys->rend_cell_auth_mac, + rcvd_mac, DIGEST256_LEN); +} + +/* Input length to KDF for key expansion */ +#define NTOR_KEY_EXPANSION_KDF_INPUT_LEN (DIGEST256_LEN + M_HSEXPAND_LEN) +/* Output length of KDF for key expansion */ +#define NTOR_KEY_EXPANSION_KDF_OUTPUT_LEN (DIGEST256_LEN*3+CIPHER256_KEY_LEN*2) + +/** Given the rendezvous key material in <b>hs_ntor_rend_cell_keys</b>, do the + * circuit key expansion as specified by section '4.2.1. Key expansion' and + * return a hs_ntor_rend_circuit_keys_t structure with the computed keys. */ +hs_ntor_rend_circuit_keys_t * +hs_ntor_circuit_key_expansion( + const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys) +{ + uint8_t *ptr; + uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN]; + uint8_t keys[NTOR_KEY_EXPANSION_KDF_OUTPUT_LEN]; + crypto_xof_t *xof; + hs_ntor_rend_circuit_keys_t *rend_circuit_keys = NULL; + + /* Let's build the input to the KDF */ + ptr = kdf_input; + APPEND(ptr, hs_ntor_rend_cell_keys->ntor_key_seed, DIGEST256_LEN); + APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND)); + tor_assert(ptr == kdf_input + sizeof(kdf_input)); + + /* Generate the keys */ + xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input)); + crypto_xof_squeeze_bytes(xof, keys, sizeof(keys)); + crypto_xof_free(xof); + + /* Generate keys structure and assign keys to it */ + rend_circuit_keys = tor_malloc_zero(sizeof(hs_ntor_rend_circuit_keys_t)); + ptr = keys; + memcpy(rend_circuit_keys->KH, ptr, DIGEST256_LEN); + ptr += DIGEST256_LEN;; + memcpy(rend_circuit_keys->Df, ptr, DIGEST256_LEN); + ptr += DIGEST256_LEN; + memcpy(rend_circuit_keys->Db, ptr, DIGEST256_LEN); + ptr += DIGEST256_LEN; + memcpy(rend_circuit_keys->Kf, ptr, CIPHER256_KEY_LEN); + ptr += CIPHER256_KEY_LEN; + memcpy(rend_circuit_keys->Kb, ptr, CIPHER256_KEY_LEN); + ptr += CIPHER256_KEY_LEN; + tor_assert(ptr == keys + sizeof(keys)); + + return rend_circuit_keys; +} + diff --git a/src/or/hs_ntor.h b/src/or/hs_ntor.h new file mode 100644 index 0000000000..cd75f46a4c --- /dev/null +++ b/src/or/hs_ntor.h @@ -0,0 +1,77 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_HS_NTOR_H +#define TOR_HS_NTOR_H + +#include "or.h" + +/* Key material needed to encode/decode INTRODUCE1 cells */ +typedef struct { + /* Key used for encryption of encrypted INTRODUCE1 blob */ + uint8_t enc_key[CIPHER256_KEY_LEN]; + /* MAC key used to protect encrypted INTRODUCE1 blob */ + uint8_t mac_key[DIGEST256_LEN]; +} hs_ntor_intro_cell_keys_t; + +/* Key material needed to encode/decode RENDEZVOUS1 cells */ +typedef struct { + /* This is the MAC of the HANDSHAKE_INFO field */ + uint8_t rend_cell_auth_mac[DIGEST256_LEN]; + /* This is the key seed used to derive further rendezvous crypto keys as + * detailed in section 4.2.1 of rend-spec-ng.txt. */ + uint8_t ntor_key_seed[DIGEST256_LEN]; +} hs_ntor_rend_cell_keys_t; + +/* Key material resulting from key expansion as detailed in section "4.2.1. Key + * expansion" of rend-spec-ng.txt. */ +typedef struct { + /* Per-circuit key material used in ESTABLISH_INTRO cell */ + uint8_t KH[DIGEST256_LEN]; + /* Authentication key for outgoing RELAY cells */ + uint8_t Df[DIGEST256_LEN]; + /* Authentication key for incoming RELAY cells */ + uint8_t Db[DIGEST256_LEN]; + /* Encryption key for outgoing RELAY cells */ + uint8_t Kf[CIPHER256_KEY_LEN]; + /* Decryption key for incoming RELAY cells */ + uint8_t Kb[CIPHER256_KEY_LEN]; +} hs_ntor_rend_circuit_keys_t; + +int hs_ntor_client_get_introduce1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_public_key_t *intro_enc_pubkey, + const curve25519_keypair_t *client_ephemeral_enc_keypair, + const uint8_t *subcredential, + hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out); + +int hs_ntor_client_get_rendezvous1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_keypair_t *client_ephemeral_enc_keypair, + const curve25519_public_key_t *intro_enc_pubkey, + const curve25519_public_key_t *service_ephemeral_rend_pubkey, + hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out); + +int hs_ntor_service_get_introduce1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_keypair_t *intro_enc_keypair, + const curve25519_public_key_t *client_ephemeral_enc_pubkey, + const uint8_t *subcredential, + hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out); + +int hs_ntor_service_get_rendezvous1_keys( + const ed25519_public_key_t *intro_auth_pubkey, + const curve25519_keypair_t *intro_enc_keypair, + const curve25519_keypair_t *service_ephemeral_rend_keypair, + const curve25519_public_key_t *client_ephemeral_enc_pubkey, + hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out); + +hs_ntor_rend_circuit_keys_t *hs_ntor_circuit_key_expansion( + const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys); + +int hs_ntor_client_rendezvous2_mac_is_good( + const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys, + const uint8_t *rcvd_mac); + +#endif + diff --git a/src/or/hs_service.c b/src/or/hs_service.c index 8687403b86..205ef11c92 100644 --- a/src/or/hs_service.c +++ b/src/or/hs_service.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,7 @@ #include "circuitlist.h" #include "circpathbias.h" +#include "hs_intropoint.h" #include "hs_service.h" #include "hs_common.h" @@ -27,7 +28,7 @@ * bytes written, or a negative integer if there was an error. */ ssize_t get_establish_intro_payload(uint8_t *buf_out, size_t buf_out_len, - const hs_cell_establish_intro_t *cell) + const trn_cell_establish_intro_t *cell) { ssize_t bytes_used = 0; @@ -35,31 +36,31 @@ get_establish_intro_payload(uint8_t *buf_out, size_t buf_out_len, return -1; } - bytes_used = hs_cell_establish_intro_encode(buf_out, buf_out_len, + bytes_used = trn_cell_establish_intro_encode(buf_out, buf_out_len, cell); return bytes_used; } /* Set the cell extensions of <b>cell</b>. */ static void -set_cell_extensions(hs_cell_establish_intro_t *cell) +set_trn_cell_extensions(trn_cell_establish_intro_t *cell) { - cell_extension_t *cell_extensions = cell_extension_new(); + trn_cell_extension_t *trn_cell_extensions = trn_cell_extension_new(); /* For now, we don't use extensions at all. */ - cell_extensions->num = 0; /* It's already zeroed, but be explicit. */ - hs_cell_establish_intro_set_extensions(cell, cell_extensions); + trn_cell_extensions->num = 0; /* It's already zeroed, but be explicit. */ + trn_cell_establish_intro_set_extensions(cell, trn_cell_extensions); } /** Given the circuit handshake info in <b>circuit_key_material</b>, create and * return an ESTABLISH_INTRO cell. Return NULL if something went wrong. The * returned cell is allocated on the heap and it's the responsibility of the * caller to free it. */ -hs_cell_establish_intro_t * +trn_cell_establish_intro_t * generate_establish_intro_cell(const uint8_t *circuit_key_material, size_t circuit_key_material_len) { - hs_cell_establish_intro_t *cell = NULL; + trn_cell_establish_intro_t *cell = NULL; ssize_t encoded_len; log_warn(LD_GENERAL, @@ -72,31 +73,32 @@ generate_establish_intro_cell(const uint8_t *circuit_key_material, goto err; } - cell = hs_cell_establish_intro_new(); + cell = trn_cell_establish_intro_new(); /* Set AUTH_KEY_TYPE: 2 means ed25519 */ - hs_cell_establish_intro_set_auth_key_type(cell, AUTH_KEY_ED25519); + trn_cell_establish_intro_set_auth_key_type(cell, + HS_INTRO_AUTH_KEY_TYPE_ED25519); /* Set AUTH_KEY_LEN field */ /* Must also set byte-length of AUTH_KEY to match */ int auth_key_len = ED25519_PUBKEY_LEN; - hs_cell_establish_intro_set_auth_key_len(cell, auth_key_len); - hs_cell_establish_intro_setlen_auth_key(cell, auth_key_len); + trn_cell_establish_intro_set_auth_key_len(cell, auth_key_len); + trn_cell_establish_intro_setlen_auth_key(cell, auth_key_len); /* Set AUTH_KEY field */ - uint8_t *auth_key_ptr = hs_cell_establish_intro_getarray_auth_key(cell); + uint8_t *auth_key_ptr = trn_cell_establish_intro_getarray_auth_key(cell); memcpy(auth_key_ptr, key_struct.pubkey.pubkey, auth_key_len); /* No cell extensions needed */ - set_cell_extensions(cell); + set_trn_cell_extensions(cell); /* Set signature size. We need to do this up here, because _encode() needs it and we need to call _encode() to calculate the MAC and signature. */ int sig_len = ED25519_SIG_LEN; - hs_cell_establish_intro_set_sig_len(cell, sig_len); - hs_cell_establish_intro_setlen_sig(cell, sig_len); + trn_cell_establish_intro_set_sig_len(cell, sig_len); + trn_cell_establish_intro_setlen_sig(cell, sig_len); /* XXX How to make this process easier and nicer? */ @@ -107,7 +109,7 @@ generate_establish_intro_cell(const uint8_t *circuit_key_material, uint8_t cell_bytes_tmp[RELAY_PAYLOAD_SIZE] = {0}; uint8_t mac[TRUNNEL_SHA3_256_LEN]; - encoded_len = hs_cell_establish_intro_encode(cell_bytes_tmp, + encoded_len = trn_cell_establish_intro_encode(cell_bytes_tmp, sizeof(cell_bytes_tmp), cell); if (encoded_len < 0) { @@ -126,7 +128,7 @@ generate_establish_intro_cell(const uint8_t *circuit_key_material, (ED25519_SIG_LEN + 2 + TRUNNEL_SHA3_256_LEN)); /* Write the MAC to the cell */ uint8_t *handshake_ptr = - hs_cell_establish_intro_getarray_handshake_mac(cell); + trn_cell_establish_intro_getarray_handshake_mac(cell); memcpy(handshake_ptr, mac, sizeof(mac)); } @@ -137,7 +139,7 @@ generate_establish_intro_cell(const uint8_t *circuit_key_material, uint8_t cell_bytes_tmp[RELAY_PAYLOAD_SIZE] = {0}; ed25519_signature_t sig; - encoded_len = hs_cell_establish_intro_encode(cell_bytes_tmp, + encoded_len = trn_cell_establish_intro_encode(cell_bytes_tmp, sizeof(cell_bytes_tmp), cell); if (encoded_len < 0) { @@ -158,7 +160,7 @@ generate_establish_intro_cell(const uint8_t *circuit_key_material, } /* And write the signature to the cell */ - uint8_t *sig_ptr = hs_cell_establish_intro_getarray_sig(cell); + uint8_t *sig_ptr = trn_cell_establish_intro_getarray_sig(cell); memcpy(sig_ptr, sig.sig, sig_len); } @@ -166,7 +168,7 @@ generate_establish_intro_cell(const uint8_t *circuit_key_material, return cell; err: - hs_cell_establish_intro_free(cell); + trn_cell_establish_intro_free(cell); return NULL; } diff --git a/src/or/hs_service.h b/src/or/hs_service.h index 5d2d8dc4bb..3302592762 100644 --- a/src/or/hs_service.h +++ b/src/or/hs_service.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,12 +16,12 @@ * hs_service.o ends up with no symbols in libor.a which makes clang throw a * warning at compile time. See #21825. */ -hs_cell_establish_intro_t * +trn_cell_establish_intro_t * generate_establish_intro_cell(const uint8_t *circuit_key_material, size_t circuit_key_material_len); ssize_t get_establish_intro_payload(uint8_t *buf, size_t buf_len, - const hs_cell_establish_intro_t *cell); + const trn_cell_establish_intro_t *cell); #endif /* TOR_HS_SERVICE_H */ diff --git a/src/or/include.am b/src/or/include.am index 4e54deca55..1ef5afa013 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -22,6 +22,7 @@ LIBTOR_A_SOURCES = \ src/or/bridges.c \ src/or/buffers.c \ src/or/channel.c \ + src/or/channelpadding.c \ src/or/channeltls.c \ src/or/circpathbias.c \ src/or/circuitbuild.c \ @@ -36,6 +37,9 @@ LIBTOR_A_SOURCES = \ src/or/connection.c \ src/or/connection_edge.c \ src/or/connection_or.c \ + src/or/conscache.c \ + src/or/consdiff.c \ + src/or/consdiffmgr.c \ src/or/control.c \ src/or/cpuworker.c \ src/or/dircollate.c \ @@ -48,6 +52,7 @@ LIBTOR_A_SOURCES = \ src/or/geoip.c \ src/or/hs_intropoint.c \ src/or/hs_circuitmap.c \ + src/or/hs_ntor.c \ src/or/hs_service.c \ src/or/entrynodes.c \ src/or/ext_orport.c \ @@ -116,8 +121,11 @@ src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libev src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ + src/trace/libor-trace.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ \ + $(rust_ldadd) if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -129,7 +137,8 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ endif ORHEADERS = \ @@ -137,6 +146,7 @@ ORHEADERS = \ src/or/bridges.h \ src/or/buffers.h \ src/or/channel.h \ + src/or/channelpadding.h \ src/or/channeltls.h \ src/or/circpathbias.h \ src/or/circuitbuild.h \ @@ -151,6 +161,9 @@ ORHEADERS = \ src/or/connection.h \ src/or/connection_edge.h \ src/or/connection_or.h \ + src/or/conscache.h \ + src/or/consdiff.h \ + src/or/consdiffmgr.h \ src/or/control.h \ src/or/cpuworker.h \ src/or/dircollate.h \ @@ -171,6 +184,7 @@ ORHEADERS = \ src/or/hs_descriptor.h \ src/or/hs_intropoint.h \ src/or/hs_circuitmap.h \ + src/or/hs_ntor.h \ src/or/hs_service.h \ src/or/keypin.h \ src/or/main.h \ diff --git a/src/or/keypin.c b/src/or/keypin.c index 2d4c4e92d2..1698dc184f 100644 --- a/src/or/keypin.c +++ b/src/or/keypin.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/keypin.h b/src/or/keypin.h index 673f24d9e3..2564f5befb 100644 --- a/src/or/keypin.h +++ b/src/or/keypin.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_KEYPIN_H diff --git a/src/or/main.c b/src/or/main.c index 3139381f30..9699c8d381 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -54,15 +54,19 @@ #include "buffers.h" #include "channel.h" #include "channeltls.h" +#include "channelpadding.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compat_rust.h" +#include "compress.h" #include "config.h" #include "confparse.h" #include "connection.h" #include "connection_edge.h" #include "connection_or.h" +#include "consdiffmgr.h" #include "control.h" #include "cpuworker.h" #include "crypto_s2k.h" @@ -104,7 +108,6 @@ #include "ext_orport.h" #ifdef USE_DMALLOC #include <dmalloc.h> -#include <openssl/crypto.h> #endif #include "memarea.h" #include "sandbox.h" @@ -176,7 +179,7 @@ static int signewnym_is_pending = 0; static unsigned newnym_epoch = 0; /** Smartlist of all open connections. */ -static smartlist_t *connection_array = NULL; +STATIC smartlist_t *connection_array = NULL; /** List of connections that have been marked for close and need to be freed * and removed from connection_array. */ static smartlist_t *closeable_connection_lst = NULL; @@ -1095,8 +1098,9 @@ run_connection_housekeeping(int i, time_t now) } else if (!have_any_circuits && now - or_conn->idle_timeout >= chan->timestamp_last_had_circuits) { - log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " - "[no circuits for %d; timeout %d; %scanonical].", + log_info(LD_OR,"Expiring non-used OR connection "U64_FORMAT" to fd %d " + "(%s:%d) [no circuits for %d; timeout %d; %scanonical].", + U64_PRINTF_ARG(chan->global_identifier), (int)conn->s, conn->address, conn->port, (int)(now - chan->timestamp_last_had_circuits), or_conn->idle_timeout, @@ -1119,6 +1123,8 @@ run_connection_housekeeping(int i, time_t now) memset(&cell,0,sizeof(cell_t)); cell.command = CELL_PADDING; connection_or_write_cell_to_buf(&cell, or_conn); + } else { + channelpadding_decide_to_pad_channel(chan); } } @@ -1161,6 +1167,7 @@ static int periodic_events_initialized = 0; #define CALLBACK(name) \ static int name ## _callback(time_t, const or_options_t *) CALLBACK(rotate_onion_key); +CALLBACK(check_onion_keys_expiry_time); CALLBACK(check_ed_keys); CALLBACK(launch_descriptor_fetches); CALLBACK(rotate_x509_certificate); @@ -1184,6 +1191,9 @@ CALLBACK(check_dns_honesty); CALLBACK(write_bridge_ns); CALLBACK(check_fw_helper_app); CALLBACK(heartbeat); +CALLBACK(clean_consdiffmgr); +CALLBACK(reset_padding_counts); +CALLBACK(check_canonical_channels); #undef CALLBACK @@ -1192,6 +1202,7 @@ CALLBACK(heartbeat); static periodic_event_item_t periodic_events[] = { CALLBACK(rotate_onion_key), + CALLBACK(check_onion_keys_expiry_time), CALLBACK(check_ed_keys), CALLBACK(launch_descriptor_fetches), CALLBACK(rotate_x509_certificate), @@ -1215,6 +1226,9 @@ static periodic_event_item_t periodic_events[] = { CALLBACK(write_bridge_ns), CALLBACK(check_fw_helper_app), CALLBACK(heartbeat), + CALLBACK(clean_consdiffmgr), + CALLBACK(reset_padding_counts), + CALLBACK(check_canonical_channels), END_OF_PERIODIC_EVENTS }; #undef CALLBACK @@ -1470,19 +1484,26 @@ run_scheduled_events(time_t now) /* 11b. check pending unconfigured managed proxies */ if (!net_is_disabled() && pt_proxies_configuration_pending()) pt_configure_remaining_proxies(); + + /* 12. launch diff computations. (This is free if there are none to + * launch.) */ + if (server_mode(options)) { + consdiffmgr_rescan(); + } } -/* Periodic callback: Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion - * keys, shut down and restart all cpuworkers, and update our descriptor if - * necessary. +/* Periodic callback: rotate the onion keys after the period defined by the + * "onion-key-rotation-days" consensus parameter, shut down and restart all + * cpuworkers, and update our descriptor if necessary. */ static int rotate_onion_key_callback(time_t now, const or_options_t *options) { if (server_mode(options)) { - time_t rotation_time = get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME; + int onion_key_lifetime = get_onion_key_lifetime(); + time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime; if (rotation_time > now) { - return safe_timer_diff(now, rotation_time); + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; } log_info(LD_GENERAL,"Rotating onion key."); @@ -1493,11 +1514,35 @@ rotate_onion_key_callback(time_t now, const or_options_t *options) } if (advertised_server_mode() && !options->DisableNetwork) router_upload_dir_desc_to_dirservers(0); - return MIN_ONION_KEY_LIFETIME; + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; } return PERIODIC_EVENT_NO_UPDATE; } +/* Period callback: Check if our old onion keys are still valid after the + * period of time defined by the consensus parameter + * "onion-key-grace-period-days", otherwise expire them by setting them to + * NULL. + */ +static int +check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options) +{ + if (server_mode(options)) { + int onion_key_grace_period = get_onion_key_grace_period(); + time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period; + if (expiry_time > now) { + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + + log_info(LD_GENERAL, "Expiring old onion keys."); + expire_old_onion_keys(); + cpuworkers_rotate_keyinfo(); + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + + return PERIODIC_EVENT_NO_UPDATE; +} + /* Periodic callback: Every 30 seconds, check whether it's time to make new * Ed25519 subkeys. */ @@ -1726,6 +1771,28 @@ write_stats_file_callback(time_t now, const or_options_t *options) return safe_timer_diff(now, next_time_to_write_stats_files); } +#define CHANNEL_CHECK_INTERVAL (60*60) +static int +check_canonical_channels_callback(time_t now, const or_options_t *options) +{ + (void)now; + if (public_server_mode(options)) + channel_check_for_duplicates(); + + return CHANNEL_CHECK_INTERVAL; +} + +static int +reset_padding_counts_callback(time_t now, const or_options_t *options) +{ + if (options->PaddingStatistics) { + rep_hist_prep_published_padding_counts(now); + } + + rep_hist_reset_padding_counts(); + return REPHIST_CELL_PADDING_COUNTS_INTERVAL; +} + /** * Periodic callback: Write bridge statistics to disk if appropriate. */ @@ -2014,6 +2081,17 @@ heartbeat_callback(time_t now, const or_options_t *options) return options->HeartbeatPeriod; } +#define CDM_CLEAN_CALLBACK_INTERVAL 600 +static int +clean_consdiffmgr_callback(time_t now, const or_options_t *options) +{ + (void)now; + if (server_mode(options)) { + consdiffmgr_cleanup(); + } + return CDM_CLEAN_CALLBACK_INTERVAL; +} + /** Timer: used to invoke second_elapsed_callback() once per second. */ static periodic_timer_t *second_timer = NULL; /** Number of libevent errors in the last second: we die if we get too many. */ @@ -2343,6 +2421,8 @@ do_main_loop(void) } handle_signals(1); + monotime_init(); + timers_initialize(); /* load the private keys, if we're supposed to have them, and set up the * TLS context. */ @@ -2410,9 +2490,10 @@ do_main_loop(void) /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } + consdiffmgr_enable_background_compression(); /* Setup shared random protocol subsystem. */ - if (authdir_mode_publishes_statuses(get_options())) { + if (authdir_mode_v3(get_options())) { if (sr_init(1) < 0) { return -1; } @@ -2980,11 +3061,16 @@ tor_init(int argc, char *argv[]) const char *version = get_version(); log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, " - "OpenSSL %s and Zlib %s.", version, + "OpenSSL %s, Zlib %s, Liblzma %s, and Libzstd %s.", version, get_uname(), tor_libevent_get_version_str(), crypto_openssl_get_version_str(), - tor_zlib_get_version_str()); + tor_compress_supports_method(ZLIB_METHOD) ? + tor_compress_version_str(ZLIB_METHOD) : "N/A", + tor_compress_supports_method(LZMA_METHOD) ? + tor_compress_version_str(LZMA_METHOD) : "N/A", + tor_compress_supports_method(ZSTD_METHOD) ? + tor_compress_version_str(ZSTD_METHOD) : "N/A"); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " @@ -2995,6 +3081,15 @@ tor_init(int argc, char *argv[]) "Expect more bugs than usual."); } + { + rust_str_t rust_str = rust_welcome_string(); + const char *s = rust_str_get(rust_str); + if (strlen(s) > 0) { + log_notice(LD_GENERAL, "%s", s); + } + rust_str_free(rust_str); + } + if (network_init()<0) { log_err(LD_BUG,"Error initializing network; exiting."); return -1; @@ -3009,6 +3104,13 @@ tor_init(int argc, char *argv[]) /* The options are now initialised */ const or_options_t *options = get_options(); + /* Initialize channelpadding parameters to defaults until we get + * a consensus */ + channelpadding_new_consensus_params(NULL); + + /* Initialize predicted ports list after loading options */ + predicted_ports_init(); + #ifndef _WIN32 if (geteuid()==0) log_warn(LD_GENERAL,"You are running Tor as root. You don't need to, " @@ -3138,6 +3240,7 @@ tor_free_all(int postfork) sandbox_free_getaddrinfo_cache(); protover_free_all(); bridges_free_all(); + consdiffmgr_free_all(); if (!postfork) { config_free_all(); or_state_free_all(); @@ -3167,6 +3270,7 @@ tor_free_all(int postfork) if (!postfork) { escaped(NULL); esc_router_info(NULL); + clean_up_backtrace_handler(); logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -3204,6 +3308,9 @@ tor_cleanup(void) rep_hist_record_mtbf_data(now, 0); keypin_close_journal(); } + + timers_shutdown(); + #ifdef USE_DMALLOC dmalloc_log_stats(); #endif @@ -3559,6 +3666,8 @@ sandbox_init_filter(void) OPEN_DATADIR("stats"); STAT_DATADIR("stats"); STAT_DATADIR2("stats", "dirreq-stats"); + + consdiffmgr_register_with_sandbox(&cfg); } init_addrinfo(); @@ -3575,6 +3684,11 @@ tor_main(int argc, char *argv[]) int result = 0; #ifdef _WIN32 +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); /* Call SetProcessDEPPolicy to permanently enable DEP. The function will not resolve on earlier versions of Windows, and failure is not dangerous. */ @@ -3583,7 +3697,10 @@ tor_main(int argc, char *argv[]) typedef BOOL (WINAPI *PSETDEP)(DWORD); PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, "SetProcessDEPPolicy"); - if (setdeppolicy) setdeppolicy(1); /* PROCESS_DEP_ENABLE */ + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } } #endif @@ -3591,14 +3708,15 @@ tor_main(int argc, char *argv[]) update_approx_time(time(NULL)); tor_threads_init(); + tor_compress_init(); init_logging(0); monotime_init(); #ifdef USE_DMALLOC { /* Instruct OpenSSL to use our internal wrappers for malloc, realloc and free. */ - int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_); - tor_assert(r); + int r = crypto_use_tor_alloc_functions(); + tor_assert(r == 0); } #endif #ifdef NT_SERVICE diff --git a/src/or/main.h b/src/or/main.h index 07b22598b1..57aa372750 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -92,6 +92,9 @@ STATIC void init_connection_lists(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); +#ifdef TOR_UNIT_TESTS +extern smartlist_t *connection_array; +#endif #endif #endif diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 140117f683..a4e6b409c4 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -804,18 +804,6 @@ microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) return md; } -/** Return the mean size of decriptors added to <b>cache</b> since it was last - * cleared. Used to estimate the size of large downloads. */ -size_t -microdesc_average_size(microdesc_cache_t *cache) -{ - if (!cache) - cache = get_microdesc_cache(); - if (!cache->n_seen) - return 512; - return (size_t)(cache->total_len_seen / cache->n_seen); -} - /** Return a smartlist of all the sha256 digest of the microdescriptors that * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers * to internals of <b>ns</b>; you should not free the members of the resulting diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 40c83139e9..943873066e 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,8 +32,6 @@ void microdesc_cache_clear(microdesc_cache_t *cache); microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d); -size_t microdesc_average_size(microdesc_cache_t *cache); - smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 1fd0772f3e..fffd1078be 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -46,6 +46,7 @@ #include "config.h" #include "connection.h" #include "connection_or.h" +#include "consdiffmgr.h" #include "control.h" #include "directory.h" #include "dirserv.h" @@ -63,6 +64,7 @@ #include "shared_random.h" #include "transports.h" #include "torcert.h" +#include "channelpadding.h" /** Map from lowercase nickname to identity digest of named server, if any. */ static strmap_t *named_server_map = NULL; @@ -72,11 +74,11 @@ static strmap_t *unnamed_server_map = NULL; /** Most recently received and validated v3 "ns"-flavored consensus network * status. */ -static networkstatus_t *current_ns_consensus = NULL; +STATIC networkstatus_t *current_ns_consensus = NULL; /** Most recently received and validated v3 "microdec"-flavored consensus * network status. */ -static networkstatus_t *current_md_consensus = NULL; +STATIC networkstatus_t *current_md_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ @@ -178,53 +180,74 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } +/** + * Read and and return the cached consensus of type <b>flavorname</b>. If + * <b>unverified</b> is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static char * +networkstatus_read_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char buf[128]; + const char *prefix; + if (unverified_consensus) { + prefix = "unverified"; + } else { + prefix = "cached"; + } + if (flav == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix); + } else { + tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); + } + + char *filename = get_datadir_fname(buf); + char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + tor_free(filename); + return result; +} + +/** Return a new string containing the current cached consensus of flavor + * <b>flavorname</b>. */ +char * +networkstatus_read_cached_consensus(const char *flavorname) + { + int flav = networkstatus_parse_flavor_name(flavorname); + if (flav < 0) + return NULL; + return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); +} + /** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { - char *filename; - char *s; const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; int flav; /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { - char buf[128]; const char *flavor = networkstatus_get_flavor_name(flav); - if (flav == FLAV_NS) { - filename = get_datadir_fname("cached-consensus"); - } else { - tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); - filename = get_datadir_fname(buf); - } - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); if (s) { if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { - log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", - flavor, filename); + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", + flavor); } tor_free(s); } - tor_free(filename); - - if (flav == FLAV_NS) { - filename = get_datadir_fname("unverified-consensus"); - } else { - tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); - filename = get_datadir_fname(buf); - } - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); if (s) { if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS, NULL)) { - log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", - flavor, filename); - } + log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " + "from cache", flavor); + } tor_free(s); } - tor_free(filename); } if (!networkstatus_get_latest_consensus()) { @@ -1384,16 +1407,24 @@ networkstatus_get_live_consensus,(time_t now)) * Return 1 if the consensus is reasonably live, or 0 if it is too old. */ int -networkstatus_consensus_reasonably_live(networkstatus_t *consensus, time_t now) +networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, + time_t now) { -#define REASONABLY_LIVE_TIME (24*60*60) if (BUG(!consensus)) return 0; - if (now <= consensus->valid_until + REASONABLY_LIVE_TIME) - return 1; + return networkstatus_valid_until_is_reasonably_live(consensus->valid_until, + now); +} - return 0; +/** As networkstatus_consensus_reasonably_live, but takes a valid_until + * time rather than an entire consensus. */ +int +networkstatus_valid_until_is_reasonably_live(time_t valid_until, + time_t now) +{ +#define REASONABLY_LIVE_TIME (24*60*60) + return (now <= valid_until + REASONABLY_LIVE_TIME); } /* XXXX remove this in favor of get_live_consensus. But actually, @@ -1966,6 +1997,7 @@ networkstatus_set_current_consensus(const char *consensus, circuit_build_times_new_consensus_params( get_circuit_build_times_mutable(), c); + channelpadding_new_consensus_params(c); } /* Reset the failure count only if this consensus is actually valid. */ @@ -1980,7 +2012,11 @@ networkstatus_set_current_consensus(const char *consensus, dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, + c->digest_sha3_as_signed, c->valid_after); + if (server_mode(get_options())) { + consdiffmgr_add_consensus(consensus, c); + } } if (!from_cache) { @@ -2441,10 +2477,8 @@ networkstatus_parse_flavor_name(const char *flavname) * running, or otherwise not a descriptor that we would make any * use of even if we had it. Else return 1. */ int -client_would_use_router(const routerstatus_t *rs, time_t now, - const or_options_t *options) +client_would_use_router(const routerstatus_t *rs, time_t now) { - (void) options; /* unused */ if (!rs->is_flagged_running) { /* If we had this router descriptor, we wouldn't even bother using it. * (Fetching and storing depends on by we_want_to_fetch_flavor().) */ diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 66cd84c88e..e774c4d266 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,6 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); +char *networkstatus_read_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free(routerstatus_t *rs); void networkstatus_vote_free(networkstatus_t *ns); @@ -75,14 +76,15 @@ int should_delay_dir_fetches(const or_options_t *options,const char **msg_out); void update_networkstatus_downloads(time_t now); void update_certificate_downloads(time_t now); int consensus_is_waiting_for_certs(void); -int client_would_use_router(const routerstatus_t *rs, time_t now, - const or_options_t *options); +int client_would_use_router(const routerstatus_t *rs, time_t now); MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void)); MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor, (consensus_flavor_t f)); MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); -int networkstatus_consensus_reasonably_live(networkstatus_t *consensus, +int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); +int networkstatus_valid_until_is_reasonably_live(time_t valid_until, + time_t now); networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, int flavor); MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now)); @@ -138,7 +140,9 @@ void vote_routerstatus_free(vote_routerstatus_t *rs); #ifdef TOR_UNIT_TESTS STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c, const char *flavor); -#endif // TOR_UNIT_TESTS +extern networkstatus_t *current_ns_consensus; +extern networkstatus_t *current_md_consensus; +#endif #endif #endif diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 96e95baf5a..0ef9741243 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -74,7 +74,6 @@ static void count_usable_descriptors(int *num_present, int *num_usable, smartlist_t *descs_out, const networkstatus_t *consensus, - const or_options_t *options, time_t now, routerset_t *in_set, usable_descriptor_t exit_only); @@ -1011,16 +1010,6 @@ node_get_platform(const node_t *node) return NULL; } -/** Return <b>node</b>'s time of publication, or 0 if we don't have one. */ -time_t -node_get_published_on(const node_t *node) -{ - if (node->ri) - return node->ri->cache_info.published_on; - else - return 0; -} - /** Return true iff <b>node</b> is one representing this router. */ int node_is_me(const node_t *node) @@ -1690,7 +1679,7 @@ static void count_usable_descriptors(int *num_present, int *num_usable, smartlist_t *descs_out, const networkstatus_t *consensus, - const or_options_t *options, time_t now, + time_t now, routerset_t *in_set, usable_descriptor_t exit_only) { @@ -1707,7 +1696,7 @@ count_usable_descriptors(int *num_present, int *num_usable, continue; if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1)) continue; - if (client_would_use_router(rs, now, options)) { + if (client_would_use_router(rs, now)) { const char * const digest = rs->descriptor_digest; int present; ++*num_usable; /* the consensus says we want it. */ @@ -1761,10 +1750,10 @@ compute_frac_paths_available(const networkstatus_t *consensus, const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, - mid, consensus, options, now, NULL, + mid, consensus, now, NULL, USABLE_DESCRIPTOR_ALL); if (options->EntryNodes) { - count_usable_descriptors(&np, &nu, guards, consensus, options, now, + count_usable_descriptors(&np, &nu, guards, consensus, now, options->EntryNodes, USABLE_DESCRIPTOR_ALL); } else { SMARTLIST_FOREACH(mid, const node_t *, node, { @@ -1785,7 +1774,7 @@ compute_frac_paths_available(const networkstatus_t *consensus, * an unavoidable feature of forcing authorities to declare * certain nodes as exits. */ - count_usable_descriptors(&np, &nu, exits, consensus, options, now, + count_usable_descriptors(&np, &nu, exits, consensus, now, NULL, USABLE_DESCRIPTOR_EXIT_ONLY); log_debug(LD_NET, "%s: %d present, %d usable", @@ -1834,7 +1823,7 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_t *myexits_unflagged = smartlist_new(); /* All nodes with exit flag in ExitNodes option */ - count_usable_descriptors(&np, &nu, myexits, consensus, options, now, + count_usable_descriptors(&np, &nu, myexits, consensus, now, options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY); log_debug(LD_NET, "%s: %d present, %d usable", @@ -1845,7 +1834,7 @@ compute_frac_paths_available(const networkstatus_t *consensus, /* Now compute the nodes in the ExitNodes option where which we don't know * what their exit policy is, or we know it permits something. */ count_usable_descriptors(&np, &nu, myexits_unflagged, - consensus, options, now, + consensus, now, options->ExitNodes, USABLE_DESCRIPTOR_ALL); log_debug(LD_NET, "%s: %d present, %d usable", diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 8456d21c6c..6c063de8a3 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -53,7 +53,6 @@ const char *node_get_platform(const node_t *node); uint32_t node_get_prim_addr_ipv4h(const node_t *node); void node_get_address_string(const node_t *node, char *cp, size_t len); long node_get_declared_uptime(const node_t *node); -time_t node_get_published_on(const node_t *node); const smartlist_t *node_get_declared_family(const node_t *node); const ed25519_public_key_t *node_get_ed25519_id(const node_t *node); int node_ed25519_id_matches(const node_t *node, diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 4c65805b32..e087bd6937 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ntmain.h b/src/or/ntmain.h index 31bf38c62c..4b771b1828 100644 --- a/src/or/ntmain.h +++ b/src/or/ntmain.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion.c b/src/or/onion.c index b3898d4085..a98b97cb1d 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion.h b/src/or/onion.h index 19e4a7c381..37a7b08cb6 100644 --- a/src/or/onion.h +++ b/src/or/onion.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index 8dcbfe22d8..146943a273 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_fast.h b/src/or/onion_fast.h index b9626002c3..b31f8e9492 100644 --- a/src/or/onion_fast.h +++ b/src/or/onion_fast.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index ded97ee73d..902260b54b 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_ntor.h b/src/or/onion_ntor.h index f637b437fd..158c499de4 100644 --- a/src/or/onion_ntor.h +++ b/src/or/onion_ntor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ONION_NTOR_H diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 2769300945..294fc0df6d 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -159,7 +159,7 @@ onion_skin_TAP_server_handshake( * big. That should be impossible. */ log_info(LD_GENERAL, "crypto_dh_get_public failed."); goto err; - /* LCOV_EXCP_STOP */ + /* LCOV_EXCL_STOP */ } key_material_len = DIGEST_LEN+key_out_len; diff --git a/src/or/onion_tap.h b/src/or/onion_tap.h index a2880f6e98..bd625231f4 100644 --- a/src/or/onion_tap.h +++ b/src/or/onion_tap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/or.h b/src/or/or.h index 50e6e3e71b..1f55b55062 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -71,10 +71,11 @@ #include "tortls.h" #include "torlog.h" #include "container.h" -#include "torgzip.h" +#include "compress.h" #include "address.h" #include "compat_libevent.h" #include "ht.h" +#include "confline.h" #include "replaycache.h" #include "crypto_curve25519.h" #include "crypto_ed25519.h" @@ -147,8 +148,27 @@ /** Maximum size of a single extrainfo document, as above. */ #define MAX_EXTRAINFO_UPLOAD_SIZE 50000 -/** How often do we rotate onion keys? */ -#define MIN_ONION_KEY_LIFETIME (7*24*60*60) +/** Minimum lifetime for an onion key in days. */ +#define MIN_ONION_KEY_LIFETIME_DAYS (1) + +/** Maximum lifetime for an onion key in days. */ +#define MAX_ONION_KEY_LIFETIME_DAYS (90) + +/** Default lifetime for an onion key in days. */ +#define DEFAULT_ONION_KEY_LIFETIME_DAYS (28) + +/** Minimum grace period for acceptance of an onion key in days. + * The maximum value is defined in proposal #274 as being the current network + * consensus parameter for "onion-key-rotation-days". */ +#define MIN_ONION_KEY_GRACE_PERIOD_DAYS (1) + +/** Default grace period for acceptance of an onion key in days. */ +#define DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS (7) + +/** How often we should check the network consensus if it is time to rotate or + * expire onion keys. */ +#define ONION_KEY_CONSENSUS_CHECK_INTERVAL (60*60) + /** How often do we rotate TLS contexts? */ #define MAX_SSL_KEY_LIFETIME_INTERNAL (2*60*60) @@ -403,12 +423,13 @@ typedef enum { #define DIR_PURPOSE_FETCH_MICRODESC 19 #define DIR_PURPOSE_MAX_ 19 -/** True iff <b>p</b> is a purpose corresponding to uploading data to a - * directory server. */ +/** True iff <b>p</b> is a purpose corresponding to uploading + * data to a directory server. */ #define DIR_PURPOSE_IS_UPLOAD(p) \ ((p)==DIR_PURPOSE_UPLOAD_DIR || \ (p)==DIR_PURPOSE_UPLOAD_VOTE || \ - (p)==DIR_PURPOSE_UPLOAD_SIGNATURES) + (p)==DIR_PURPOSE_UPLOAD_SIGNATURES || \ + (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2) #define EXIT_PURPOSE_MIN_ 1 /** This exit stream wants to do an ordinary connect. */ @@ -875,6 +896,7 @@ typedef enum { #define CELL_RELAY_EARLY 9 #define CELL_CREATE2 10 #define CELL_CREATED2 11 +#define CELL_PADDING_NEGOTIATE 12 #define CELL_VPADDING 128 #define CELL_CERTS 129 @@ -1549,10 +1571,6 @@ typedef struct or_connection_t { * NETINFO cell listed the address we're connected to as recognized. */ unsigned int is_canonical:1; - /** True iff we have decided that the other end of this connection - * is a client. Connections with this flag set should never be used - * to satisfy an EXTEND request. */ - unsigned int is_connection_with_client:1; /** True iff this is an outgoing connection. */ unsigned int is_outgoing:1; unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */ @@ -1738,14 +1756,6 @@ typedef struct entry_connection_t { unsigned int is_socks_socket:1; } entry_connection_t; -typedef enum { - DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, - DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, - DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ -} dir_spool_source_t; -#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t) - /** Subtype of connection_t for an "directory connection" -- that is, an HTTP * connection to retrieve or serve directory material. */ typedef struct dir_connection_t { @@ -1760,23 +1770,15 @@ typedef struct dir_connection_t { char *requested_resource; unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */ - /* Used only for server sides of some dir connections, to implement - * "spooling" of directory material to the outbuf. Otherwise, we'd have - * to append everything to the outbuf in one enormous chunk. */ - /** What exactly are we spooling right now? */ - dir_spool_source_bitfield_t dir_spool_src : 3; - /** If we're fetching descriptors, what router purpose shall we assign * to them? */ uint8_t router_purpose; - /** List of fingerprints for networkstatuses or descriptors to be spooled. */ - smartlist_t *fingerprint_stack; - /** A cached_dir_t object that we're currently spooling out */ - struct cached_dir_t *cached_dir; - /** The current offset into cached_dir. */ - off_t cached_dir_offset; - /** The zlib object doing on-the-fly compression for spooled data. */ - tor_zlib_state_t *zlib_state; + + /** List of spooled_resource_t for objects that we're spooling. We use + * it from back to front. */ + smartlist_t *spool; + /** The compression object doing on-the-fly compression for spooled data. */ + tor_compress_state_t *compress_state; /** What rendezvous service are we querying for? */ rend_data_t *rend_data; @@ -1792,6 +1794,14 @@ typedef struct dir_connection_t { * that's going away and being used on channels instead. The dirserver still * needs this for the incoming side, so it's moved here. */ uint64_t dirreq_id; + +#ifdef MEASUREMENTS_21206 + /** Number of RELAY_DATA cells received. */ + uint32_t data_cells_received; + + /** Number of RELAY_DATA cells sent. */ + uint32_t data_cells_sent; +#endif } dir_connection_t; /** Subtype of connection_t for an connection to a controller. */ @@ -1930,11 +1940,13 @@ typedef struct addr_policy_t { * compressed form. */ typedef struct cached_dir_t { char *dir; /**< Contents of this object, NUL-terminated. */ - char *dir_z; /**< Compressed contents of this object. */ + char *dir_compressed; /**< Compressed contents of this object. */ size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ - size_t dir_z_len; /**< Length of <b>dir_z</b>. */ + size_t dir_compressed_len; /**< Length of <b>dir_compressed</b>. */ time_t published; /**< When was this object published. */ common_digests_t digests; /**< Digests of this object (networkstatus only) */ + /** Sha3 digest (also ns only) */ + uint8_t digest_sha3_as_signed[DIGEST256_LEN]; int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -2272,6 +2284,16 @@ typedef struct routerstatus_t { * ed25519 identity keys on a link handshake. */ unsigned int supports_ed25519_link_handshake:1; + /** True iff this router has a protocol list that allows it to be an + * introduction point supporting ed25519 authentication key which is part of + * the v3 protocol detailed in proposal 224. This requires HSIntro=4. */ + unsigned int supports_ed25519_hs_intro : 1; + + /** True iff this router has a protocol list that allows it to be an hidden + * service directory supporting version 3 as seen in proposal 224. This + * requires HSDir=2. */ + unsigned int supports_v3_hsdir : 1; + unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with @@ -2625,6 +2647,9 @@ typedef struct networkstatus_t { /** Digests of this document, as signed. */ common_digests_t digests; + /** A SHA3-256 digest of the document, not including signatures: used for + * consensus diffs */ + uint8_t digest_sha3_as_signed[DIGEST256_LEN]; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -3058,6 +3083,13 @@ typedef struct circuit_t { * circuit's queues; used only if CELL_STATS events are enabled and * cleared after being sent to control port. */ smartlist_t *testing_cell_stats; + + /** If set, points to an HS token that this circuit might be carrying. + * Used by the HS circuitmap. */ + hs_token_t *hs_token; + /** Hashtable node: used to look up the circuit by its HS token using the HS + circuitmap. */ + HT_ENTRY(circuit_t) hs_circuitmap_node; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -3306,6 +3338,13 @@ typedef struct origin_circuit_t { * adjust_exit_policy_from_exitpolicy_failure. */ smartlist_t *prepend_policy; + + /** How long do we wait before closing this circuit if it remains + * completely idle after it was built, in seconds? This value + * is randomized on a per-circuit basis from CircuitsAvailableTimoeut + * to 2*CircuitsAvailableTimoeut. */ + int circuit_idle_timeout; + } origin_circuit_t; struct onion_queue_t; @@ -3367,13 +3406,6 @@ typedef struct or_circuit_t { * is not marked for close. */ struct or_circuit_t *rend_splice; - /** If set, points to an HS token that this circuit might be carrying. - * Used by the HS circuitmap. */ - hs_token_t *hs_token; - /** Hashtable node: used to look up the circuit by its HS token using the HS - circuitmap. */ - HT_ENTRY(or_circuit_t) hs_circuitmap_node; - /** Stores KH for the handshake. */ char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */ @@ -3454,15 +3486,6 @@ static inline const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT( return DOWNCAST(origin_circuit_t, x); } -/** Bitfield type: things that we're willing to use invalid routers for. */ -typedef enum invalid_router_usage_t { - ALLOW_INVALID_ENTRY =1, - ALLOW_INVALID_EXIT =2, - ALLOW_INVALID_MIDDLE =4, - ALLOW_INVALID_RENDEZVOUS =8, - ALLOW_INVALID_INTRODUCTION=16, -} invalid_router_usage_t; - /* limits for TCP send and recv buffer size used for constrained sockets */ #define MIN_CONSTRAINED_TCP_BUFFER 2048 #define MAX_CONSTRAINED_TCP_BUFFER 262144 /* 256k */ @@ -3524,27 +3547,6 @@ typedef struct port_cfg_t { char unix_addr[FLEXIBLE_ARRAY_MEMBER]; } port_cfg_t; -/** Ordinary configuration line. */ -#define CONFIG_LINE_NORMAL 0 -/** Appends to previous configuration for the same option, even if we - * would ordinary replace it. */ -#define CONFIG_LINE_APPEND 1 -/* Removes all previous configuration for an option. */ -#define CONFIG_LINE_CLEAR 2 - -/** A linked list of lines in a config file. */ -typedef struct config_line_t { - char *key; - char *value; - struct config_line_t *next; - /** What special treatment (if any) does this line require? */ - unsigned int command:2; - /** If true, subsequent assignments to this linelist should replace - * it, not extend it. Set only on the first item in a linelist in an - * or_options_t. */ - unsigned int fragile:1; -} config_line_t; - typedef struct routerset_t routerset_t; /** A magic value for the (Socks|OR|...)Port options below, telling Tor @@ -3609,10 +3611,6 @@ typedef struct { int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our * process for all current and future memory. */ - /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ - smartlist_t *AllowInvalidNodes; - /** Bitmask; derived from AllowInvalidNodes. */ - invalid_router_usage_t AllowInvalid_; config_line_t *ExitPolicy; /**< Lists of exit policy components. */ int ExitPolicyRejectPrivate; /**< Should we not exit to reserved private * addresses, and our own published addresses? @@ -3623,21 +3621,6 @@ typedef struct { * configured ports. */ config_line_t *SocksPolicy; /**< Lists of socks policy components */ config_line_t *DirPolicy; /**< Lists of dir policy components */ - /** Addresses to bind for listening for SOCKS connections. */ - config_line_t *SocksListenAddress; - /** Addresses to bind for listening for transparent pf/netfilter - * connections. */ - config_line_t *TransListenAddress; - /** Addresses to bind for listening for transparent natd connections */ - config_line_t *NATDListenAddress; - /** Addresses to bind for listening for SOCKS connections. */ - config_line_t *DNSListenAddress; - /** Addresses to bind for listening for OR connections. */ - config_line_t *ORListenAddress; - /** Addresses to bind for listening for directory connections. */ - config_line_t *DirListenAddress; - /** Addresses to bind for listening for control connections. */ - config_line_t *ControlListenAddress; /** Local address to bind outbound sockets */ config_line_t *OutboundBindAddress; /** Local address to bind outbound relay sockets */ @@ -3659,7 +3642,6 @@ typedef struct { /** Whether routers accept EXTEND cells to routers with private IPs. */ int ExtendAllowPrivateAddresses; char *User; /**< Name of user to run Tor as. */ - char *Group; /**< Name of group to run Tor as. */ config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */ /** Ports to listen on for extended OR connections. */ config_line_t *ExtORPort_lines; @@ -3762,6 +3744,15 @@ typedef struct { int AvoidDiskWrites; /**< Boolean: should we never cache things to disk? * Not used yet. */ int ClientOnly; /**< Boolean: should we never evolve into a server role? */ + + int ReducedConnectionPadding; /**< Boolean: Should we try to keep connections + open shorter and pad them less against + connection-level traffic analysis? */ + /** Autobool: if auto, then connection padding will be negotiated by client + * and server. If 0, it will be fully disabled. If 1, the client will still + * pad to the server regardless of server support. */ + int ConnectionPadding; + /** To what authority types do we publish our descriptor? Choices are * "v1", "v2", "v3", "bridge", or "". */ smartlist_t *PublishServerDescriptor; @@ -3787,15 +3778,6 @@ typedef struct { /** A routerset that should be used when picking RPs for HS circuits. */ routerset_t *Tor2webRendezvousPoints; - /** Close hidden service client circuits immediately when they reach - * the normal circuit-build timeout, even if they have already sent - * an INTRODUCE1 cell on its way to the service. */ - int CloseHSClientCircuitsImmediatelyOnTimeout; - - /** Close hidden-service-side rendezvous circuits immediately when - * they reach the normal circuit-build timeout. */ - int CloseHSServiceRendCircuitsImmediatelyOnTimeout; - /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct) * circuits between the onion service server, and the introduction and * rendezvous points. (Onion service descriptors are still posted using @@ -3876,8 +3858,8 @@ typedef struct { int CircuitBuildTimeout; /**< Cull non-open circuits that were born at * least this many seconds ago. Used until * adaptive algorithm learns a new value. */ - int CircuitIdleTimeout; /**< Cull open clean circuits that were born - * at least this many seconds ago. */ + int CircuitsAvailableTimeout; /**< Try to have an open circuit for at + least this long after last activity */ int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits * and try a new circuit if the stream has been * waiting for this many seconds. If zero, use @@ -3887,10 +3869,6 @@ typedef struct { * a new one? */ int MaxCircuitDirtiness; /**< Never use circs that were first used more than this interval ago. */ - int PredictedPortsRelevanceTime; /** How long after we've requested a - * connection for a given port, do we want - * to continue to pick exits that support - * that port? */ uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing * to use in a second? */ uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing @@ -3904,8 +3882,6 @@ typedef struct { uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ int NumCPUs; /**< How many CPUs should we try to use? */ -//int RunTesting; /**< If true, create testing circuits to measure how well the -// * other ORs are running. */ config_line_t *RendConfigLines; /**< List of configuration lines * for rendezvous services. */ config_line_t *HidServAuth; /**< List of configuration lines for client-side @@ -3956,7 +3932,8 @@ typedef struct { /** If set, use these bridge authorities and not the default one. */ config_line_t *AlternateBridgeAuthority; - char *MyFamily; /**< Declared family for this OR. */ + config_line_t *MyFamily_lines; /**< Declared family for this OR. */ + config_line_t *MyFamily; /**< Declared family for this OR, normalized */ config_line_t *NodeFamilies; /**< List of config lines for * node families */ smartlist_t *NodeFamilySets; /**< List of parsed NodeFamilies values. */ @@ -4075,8 +4052,6 @@ typedef struct { int NumDirectoryGuards; /**< How many dir guards do we try to establish? * If 0, use value from NumEntryGuards. */ int RephistTrackTime; /**< How many seconds do we keep rephist info? */ - int FastFirstHopPK; /**< If Tor believes it is safe, should we save a third - * of our PK time by sending CREATE_FAST cells? */ /** Should we always fetch our dir info on the mirror schedule (which * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; @@ -4132,16 +4107,6 @@ typedef struct { * if we are a cache). For authorities, this is always true. */ int DownloadExtraInfo; - /** If true, and we are acting as a relay, allow exit circuits even when - * we are the first hop of a circuit. */ - int AllowSingleHopExits; - /** If true, don't allow relays with AllowSingleHopExits=1 to be used in - * circuits that we build. */ - int ExcludeSingleHopRelays; - /** If true, and the controller tells us to use a one-hop circuit, and the - * exit allows it, we use it. */ - int AllowSingleHopCircuits; - /** If true, we convert "www.google.com.foo.exit" addresses on the * socks/trans/natd ports into "www.google.com" addresses that * exit from the node "foo". Disabled by default since attacking @@ -4149,10 +4114,6 @@ typedef struct { * selection. */ int AllowDotExit; - /** If true, we will warn if a user gives us only an IP address - * instead of a hostname. */ - int WarnUnsafeSocks; - /** If true, we're configured to collect statistics on clients * requesting network statuses from us as directory. */ int DirReqStatistics_option; @@ -4169,6 +4130,9 @@ typedef struct { /** If true, the user wants us to collect cell statistics. */ int CellStatistics; + /** If true, the user wants us to collect padding statistics. */ + int PaddingStatistics; + /** If true, the user wants us to collect statistics as entry node. */ int EntryStatistics; @@ -4376,8 +4340,7 @@ typedef struct { int TestingDirAuthVoteGuardIsStrict; /** Relays in a testing network which should be voted HSDir - * regardless of uptime and DirPort. - * Respects VoteOnHidServDirectoriesV2. */ + * regardless of uptime and DirPort. */ routerset_t *TestingDirAuthVoteHSDir; int TestingDirAuthVoteHSDirIsStrict; @@ -4509,8 +4472,6 @@ typedef struct { int IPv6Exit; /**< Do we support exiting to IPv6 addresses? */ - char *TLSECGroup; /**< One of "P256", "P224", or nil for auto */ - /** Fraction: */ double PathsNeededToBuildCircuits; @@ -4594,6 +4555,9 @@ typedef struct { * do we enforce Ed25519 identity match? */ /* NOTE: remove this option someday. */ int AuthDirTestEd25519LinkKeys; + + /** Bool (default: 0): Tells if a %include was used on torrc */ + int IncludeUsed; } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -4819,7 +4783,7 @@ typedef uint32_t build_time_t; double circuit_build_times_quantile_cutoff(void); /** How often in seconds should we build a test circuit */ -#define CBT_DEFAULT_TEST_FREQUENCY 60 +#define CBT_DEFAULT_TEST_FREQUENCY 10 #define CBT_MIN_TEST_FREQUENCY 1 #define CBT_MAX_TEST_FREQUENCY INT32_MAX @@ -5308,7 +5272,8 @@ typedef struct dir_server_t { * address information from published? */ routerstatus_t fake_status; /**< Used when we need to pass this trusted - * dir_server_t to directory_initiate_command_* + * dir_server_t to + * directory_request_set_routerstatus. * as a routerstatus_t. Not updated by the * router-status management code! **/ @@ -5361,7 +5326,6 @@ typedef enum { CRN_NEED_UPTIME = 1<<0, CRN_NEED_CAPACITY = 1<<1, CRN_NEED_GUARD = 1<<2, - CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5, CRN_NEED_DESC = 1<<6, @@ -5376,8 +5340,6 @@ typedef enum { typedef enum was_router_added_t { /* Router was added successfully. */ ROUTER_ADDED_SUCCESSFULLY = 1, - /* Router descriptor was added with warnings to submitter. */ - ROUTER_ADDED_NOTIFY_GENERATOR = 0, /* Extrainfo document was rejected because no corresponding router * descriptor was found OR router descriptor was rejected because * it was incompatible with its extrainfo document. */ diff --git a/src/or/parsecommon.c b/src/or/parsecommon.c index ec2cec69f7..7959867875 100644 --- a/src/or/parsecommon.c +++ b/src/or/parsecommon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/parsecommon.h b/src/or/parsecommon.h index 15e9f7ae85..b9f1613457 100644 --- a/src/or/parsecommon.h +++ b/src/or/parsecommon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -157,12 +157,18 @@ typedef enum { R3_SUPERENCRYPTED, R3_SIGNATURE, R3_CREATE2_FORMATS, - R3_AUTHENTICATION_REQUIRED, + R3_INTRO_AUTH_REQUIRED, R3_SINGLE_ONION_SERVICE, R3_INTRODUCTION_POINT, R3_INTRO_AUTH_KEY, R3_INTRO_ENC_KEY, - R3_INTRO_ENC_KEY_CERTIFICATION, + R3_INTRO_ENC_KEY_CERT, + R3_INTRO_LEGACY_KEY, + R3_INTRO_LEGACY_KEY_CERT, + R3_DESC_AUTH_TYPE, + R3_DESC_AUTH_KEY, + R3_DESC_AUTH_CLIENT, + R3_ENCRYPTED, R_IPO_IDENTIFIER, R_IPO_IP_ADDRESS, diff --git a/src/or/periodic.c b/src/or/periodic.c index d02d4a7bbb..6896b41c86 100644 --- a/src/or/periodic.c +++ b/src/or/periodic.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/periodic.h b/src/or/periodic.h index 021bb4ef5c..88d00cc7e9 100644 --- a/src/or/periodic.h +++ b/src/or/periodic.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PERIODIC_H diff --git a/src/or/policies.c b/src/or/policies.c index 2aa6373f3e..3d49a6110c 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/policies.h b/src/or/policies.h index f73f850c21..ce08d497e9 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/protover.c b/src/or/protover.c index 88d549ab35..1a3e69be10 100644 --- a/src/or/protover.c +++ b/src/or/protover.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -288,7 +288,7 @@ protover_get_supported_protocols(void) return "Cons=1-2 " "Desc=1-2 " - "DirCache=1 " + "DirCache=1-2 " "HSDir=1-2 " "HSIntro=3-4 " "HSRend=1-2 " diff --git a/src/or/protover.h b/src/or/protover.h index 5c658931ea..22667bed79 100644 --- a/src/or/protover.h +++ b/src/or/protover.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/reasons.c b/src/or/reasons.c index a1566e2299..e6c325f1b3 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/reasons.h b/src/or/reasons.h index 2e12c93728..1cadf4e89e 100644 --- a/src/or/reasons.h +++ b/src/or/reasons.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/relay.c b/src/or/relay.c index 28bf32af0d..0ff53ed5e9 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -54,6 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -74,6 +75,7 @@ #include "routerlist.h" #include "routerparse.h" #include "scheduler.h" +#include "rephist.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -196,6 +198,82 @@ relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in, return 0; } +/** + * Update channel usage state based on the type of relay cell and + * circuit properties. + * + * This is needed to determine if a client channel is being + * used for application traffic, and if a relay channel is being + * used for multihop circuits and application traffic. The decision + * to pad in channelpadding.c depends upon this info (as well as + * consensus parameters) to decide what channels to pad. + */ +static void +circuit_update_channel_usage(circuit_t *circ, cell_t *cell) +{ + if (CIRCUIT_IS_ORIGIN(circ)) { + /* + * The client state was first set much earlier in + * circuit_send_next_onion_skin(), so we can start padding as early as + * possible. + * + * However, if padding turns out to be expensive, we may want to not do + * it until actual application traffic starts flowing (which is controlled + * via consensus param nf_pad_before_usage). + * + * So: If we're an origin circuit and we've created a full length circuit, + * then any CELL_RELAY cell means application data. Increase the usage + * state of the channel to indicate this. + * + * We want to wait for CELL_RELAY specifically here, so we know that + * the channel was definitely being used for data and not for extends. + * By default, we pad as soon as a channel has been used for *any* + * circuits, so this state is irrelevant to the padding decision in + * the default case. However, if padding turns out to be expensive, + * we would like the ability to avoid padding until we're absolutely + * sure that a channel is used for enough application data to be worth + * padding. + * + * (So it does not matter that CELL_RELAY_EARLY can actually contain + * application data. This is only a load reducing option and that edge + * case does not matter if we're desperately trying to reduce overhead + * anyway. See also consensus parameter nf_pad_before_usage). + */ + if (BUG(!circ->n_chan)) + return; + + if (circ->n_chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS && + cell->command == CELL_RELAY) { + circ->n_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC; + } + } else { + /* If we're a relay circuit, the question is more complicated. Basically: + * we only want to pad connections that carry multihop (anonymous) + * circuits. + * + * We assume we're more than one hop if either the previous hop + * is not a client, or if the previous hop is a client and there's + * a next hop. Then, circuit traffic starts at RELAY_EARLY, and + * user application traffic starts when we see RELAY cells. + */ + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + + if (BUG(!or_circ->p_chan)) + return; + + if (!channel_is_client(or_circ->p_chan) || + (channel_is_client(or_circ->p_chan) && circ->n_chan)) { + if (cell->command == CELL_RELAY_EARLY) { + if (or_circ->p_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS) { + or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS; + } + } else if (cell->command == CELL_RELAY) { + or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC; + } + } + } +} + /** Receive a relay cell: * - Crypt it (encrypt if headed toward the origin or if we <b>are</b> the * origin; decrypt if we're headed toward the exit). @@ -225,10 +303,13 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) { - log_warn(LD_BUG,"relay crypt failed. Dropping connection."); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "relay crypt failed. Dropping connection."); return -END_CIRC_REASON_INTERNAL; } + circuit_update_channel_usage(circ, cell); + if (recognized) { edge_connection_t *conn = NULL; @@ -257,8 +338,13 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, log_debug(LD_OR,"Sending to origin."); if ((reason = connection_edge_process_relay_cell(cell, circ, conn, layer_hint)) < 0) { - log_warn(LD_OR, - "connection_edge_process_relay_cell (at origin) failed."); + /* If a client is trying to connect to unknown hidden service port, + * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit. + * Do not log warn as this is an expected behavior for a service. */ + if (reason != END_CIRC_AT_ORIGIN) { + log_warn(LD_OR, + "connection_edge_process_relay_cell (at origin) failed."); + } return reason; } } @@ -632,6 +718,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + if (relay_command == RELAY_COMMAND_DROP) + rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) @@ -732,6 +821,16 @@ connection_edge_send_command(edge_connection_t *fromconn, return -1; } +#ifdef MEASUREMENTS_21206 + /* Keep track of the number of RELAY_DATA cells sent for directory + * connections. */ + connection_t *linked_conn = TO_CONN(fromconn)->linked_conn; + + if (linked_conn && linked_conn->type == CONN_TYPE_DIR) { + ++(TO_DIR_CONN(linked_conn)->data_cells_sent); + } +#endif + return relay_send_command_from_edge(fromconn->stream_id, circ, relay_command, payload, payload_len, cpath_layer); @@ -895,6 +994,7 @@ connection_ap_process_end_not_open( break; /* break means it'll close, below */ /* Else fall through: expire this circuit, clear the * chosen_exit_name field, and try again. */ + /* Falls through. */ case END_STREAM_REASON_RESOLVEFAILED: case END_STREAM_REASON_TIMEOUT: case END_STREAM_REASON_MISC: @@ -1513,6 +1613,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, switch (rh.command) { case RELAY_COMMAND_DROP: + rep_hist_padding_count_read(PADDING_TYPE_DROP); // log_info(domain,"Got a relay-level padding cell. Dropping."); return 0; case RELAY_COMMAND_BEGIN: @@ -1586,6 +1687,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, connection_write_to_buf((char*)(cell->payload + RELAY_HEADER_SIZE), rh.length, TO_CONN(conn)); +#ifdef MEASUREMENTS_21206 + /* Count number of RELAY_DATA cells received on a linked directory + * connection. */ + connection_t *linked_conn = TO_CONN(conn)->linked_conn; + + if (linked_conn && linked_conn->type == CONN_TYPE_DIR) { + ++(TO_DIR_CONN(linked_conn)->data_cells_received); + } +#endif + if (!optimistic_data) { /* Only send a SENDME if we're not getting optimistic data; otherwise * a SENDME could arrive before the CONNECTED. @@ -2429,7 +2540,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); - alloc += tor_zlib_get_total_allocation(); + alloc += tor_compress_get_total_allocation(); const size_t rend_cache_total = rend_cache_get_total_allocation(); alloc += rend_cache_total; if (alloc >= get_options()->MaxMemInQueues_low_threshold) { diff --git a/src/or/relay.h b/src/or/relay.h index 3acf3ee0e3..a160cd5551 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcache.c b/src/or/rendcache.c index 12c23ea87c..11b60b36a1 100644 --- a/src/or/rendcache.c +++ b/src/or/rendcache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcache.h b/src/or/rendcache.h index 746f142fcc..1bd3be2243 100644 --- a/src/or/rendcache.h +++ b/src/or/rendcache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 06744ad795..9bc2d6289d 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -724,6 +724,9 @@ directory_get_from_hs_dir(const char *desc_id, hs_dir = pick_hsdir(desc_id, desc_id_base32); if (!hs_dir) { /* No suitable hs dir can be found, stop right now. */ + control_event_hs_descriptor_failed(rend_query, NULL, "QUERY_NO_HSDIR"); + control_event_hs_descriptor_content(rend_data_get_address(rend_query), + desc_id_base32, NULL, NULL); return 0; } } @@ -744,6 +747,9 @@ directory_get_from_hs_dir(const char *desc_id, REND_DESC_COOKIE_LEN, 0)<0) { log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); + control_event_hs_descriptor_failed(rend_query, hsdir_fp, "BAD_DESC"); + control_event_hs_descriptor_content(rend_data_get_address(rend_query), + desc_id_base32, hsdir_fp, NULL); return 0; } /* Remove == signs. */ @@ -756,13 +762,15 @@ directory_get_from_hs_dir(const char *desc_id, /* Send fetch request. (Pass query and possibly descriptor cookie so that * they can be written to the directory connection and be referred to when * the response arrives. */ - directory_initiate_command_routerstatus_rend(hs_dir, - DIR_PURPOSE_FETCH_RENDDESC_V2, - ROUTER_PURPOSE_GENERAL, - how_to_fetch, - desc_id_base32, - NULL, 0, 0, - rend_query, NULL); + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_RENDDESC_V2); + directory_request_set_routerstatus(req, hs_dir); + directory_request_set_indirection(req, how_to_fetch); + directory_request_set_resource(req, desc_id_base32); + directory_request_set_rend_query(req, rend_query); + directory_initiate_request(req); + directory_request_free(req); + log_info(LD_REND, "Sending fetch request for v2 descriptor for " "service '%s' with descriptor ID '%s', auth type %d, " "and descriptor cookie '%s' to hidden service " diff --git a/src/or/rendclient.h b/src/or/rendclient.h index 164305a773..ff0f4084fd 100644 --- a/src/or/rendclient.h +++ b/src/or/rendclient.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index bc53762fb6..e1236bdd0f 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index 942ace5761..94c2480d86 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 57c8cfac92..23c3deddaa 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -96,7 +96,8 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request, /* Close any other intro circuits with the same pk. */ c = NULL; - while ((c = hs_circuitmap_get_intro_circ_v2((const uint8_t *)pk_digest))) { + while ((c = hs_circuitmap_get_intro_circ_v2_relay_side( + (const uint8_t *)pk_digest))) { log_info(LD_REND, "Replacing old circuit for service %s", safe_str(serviceid)); circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED); @@ -111,7 +112,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request, /* Now, set up this circuit. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT); - hs_circuitmap_register_intro_circ_v2(circ, (uint8_t *)pk_digest); + hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest); log_info(LD_REND, "Established introduction point on circuit %u for service %s", @@ -165,7 +166,8 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request, /* The first 20 bytes are all we look at: they have a hash of the service's * PK. */ - intro_circ = hs_circuitmap_get_intro_circ_v2((const uint8_t*)request); + intro_circ = hs_circuitmap_get_intro_circ_v2_relay_side( + (const uint8_t*)request); if (!intro_circ) { log_info(LD_REND, "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; " @@ -242,7 +244,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - if (hs_circuitmap_get_rend_circ(request)) { + if (hs_circuitmap_get_rend_circ_relay_side(request)) { log_warn(LD_PROTOCOL, "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS."); goto err; @@ -258,7 +260,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, } circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING); - hs_circuitmap_register_rend_circ(circ, request); + hs_circuitmap_register_rend_circ_relay_side(circ, request); base16_encode(hexid,9,(char*)request,4); @@ -307,7 +309,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, "Got request for rendezvous from circuit %u to cookie %s.", (unsigned)circ->p_circ_id, hexid); - rend_circ = hs_circuitmap_get_rend_circ(request); + rend_circ = hs_circuitmap_get_rend_circ_relay_side(request); if (!rend_circ) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.", @@ -342,7 +344,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); circuit_change_purpose(TO_CIRCUIT(rend_circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); - hs_circuitmap_remove_circuit(circ); + hs_circuitmap_remove_circuit(TO_CIRCUIT(circ)); rend_circ->rend_splice = circ; circ->rend_splice = rend_circ; diff --git a/src/or/rendmid.h b/src/or/rendmid.h index 347d745853..daf9e2885e 100644 --- a/src/or/rendmid.h +++ b/src/or/rendmid.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendservice.c b/src/or/rendservice.c index fb777ca154..f3b78c4663 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -76,13 +76,11 @@ static ssize_t rend_service_parse_intro_for_v3( static int rend_service_check_private_dir(const or_options_t *options, const rend_service_t *s, int create); -static int rend_service_check_private_dir_impl(const or_options_t *options, - const rend_service_t *s, - int create); static const smartlist_t* rend_get_service_list( const smartlist_t* substitute_service_list); static smartlist_t* rend_get_service_list_mutable( smartlist_t* substitute_service_list); +static int rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -100,23 +98,6 @@ struct rend_service_port_config_s { char unix_addr[FLEXIBLE_ARRAY_MEMBER]; }; -/** Try to maintain this many intro points per service by default. */ -#define NUM_INTRO_POINTS_DEFAULT 3 -/** Maximum number of intro points per service. */ -#define NUM_INTRO_POINTS_MAX 10 -/** Number of extra intro points we launch if our set of intro nodes is - * empty. See proposal 155, section 4. */ -#define NUM_INTRO_POINTS_EXTRA 2 - -/** If we can't build our intro circuits, don't retry for this long. */ -#define INTRO_CIRC_RETRY_PERIOD (60*5) -/** How many times will a hidden service operator attempt to connect to - * a requested rendezvous point before giving up? */ -#define MAX_REND_FAILURES 1 -/** How many seconds should we spend trying to connect to a requested - * rendezvous point before giving up? */ -#define MAX_REND_TIMEOUT 30 - /* Hidden service directory file names: * new file names should be added to rend_service_add_filenames_to_list() * for sandboxing purposes. */ @@ -125,9 +106,12 @@ static const char *hostname_fname = "hostname"; static const char *client_keys_fname = "client_keys"; static const char *sos_poison_fname = "onion_service_non_anonymous"; -/** A list of rend_service_t's for services run on this OP. - */ +/** A list of rend_service_t's for services run on this OP. */ static smartlist_t *rend_service_list = NULL; +/** A list of rend_service_t's for services run on this OP which is used as a + * staging area before they are put in the main list in order to prune dying + * service on config reload. */ +static smartlist_t *rend_service_staging_list = NULL; /* Like rend_get_service_list_mutable, but returns a read-only list. */ static const smartlist_t* @@ -261,35 +245,23 @@ rend_service_free_all(void) rend_service_list = NULL; } -/** Validate <b>service</b> and add it to <b>service_list</b>, or to - * the global rend_service_list if <b>service_list</b> is NULL. - * Return 0 on success. On failure, free <b>service</b> and return -1. - * Takes ownership of <b>service</b>. - */ +/* Validate a <b>service</b>. Use the <b>service_list</b> to make sure there + * is no duplicate entry for the given service object. Return 0 if valid else + * -1 if not.*/ static int -rend_add_service(smartlist_t *service_list, rend_service_t *service) +rend_validate_service(const smartlist_t *service_list, + const rend_service_t *service) { - int i; - rend_service_port_config_t *p; + int dupe = 0; + tor_assert(service_list); tor_assert(service); - smartlist_t *s_list = rend_get_service_list_mutable(service_list); - /* We must have a service list, even if it's a temporary one, so we can - * check for duplicate services */ - if (BUG(!s_list)) { - return -1; - } - - service->intro_nodes = smartlist_new(); - service->expiring_nodes = smartlist_new(); - if (service->max_streams_per_circuit < 0) { log_warn(LD_CONFIG, "Hidden service (%s) configured with negative max " "streams per circuit.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; + goto invalid; } if (service->max_streams_close_circuit < 0 || @@ -297,87 +269,107 @@ rend_add_service(smartlist_t *service_list, rend_service_t *service) log_warn(LD_CONFIG, "Hidden service (%s) configured with invalid " "max streams handling.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; + goto invalid; } if (service->auth_type != REND_NO_AUTH && - (!service->clients || - smartlist_len(service->clients) == 0)) { - log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but no " - "clients.", + (!service->clients || smartlist_len(service->clients) == 0)) { + log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but " + "no clients.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; + goto invalid; } if (!service->ports || !smartlist_len(service->ports)) { log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; - } else { - int dupe = 0; - /* XXX This duplicate check has two problems: - * - * a) It's O(n^2), but the same comment from the bottom of - * rend_config_services() should apply. - * - * b) We only compare directory paths as strings, so we can't - * detect two distinct paths that specify the same directory - * (which can arise from symlinks, case-insensitivity, bind - * mounts, etc.). - * - * It also can't detect that two separate Tor instances are trying - * to use the same HiddenServiceDir; for that, we would need a - * lock file. But this is enough to detect a simple mistake that - * at least one person has actually made. - */ - tor_assert(s_list); - if (!rend_service_is_ephemeral(service)) { - /* Skip dupe for ephemeral services. */ - SMARTLIST_FOREACH(s_list, rend_service_t*, ptr, - dupe = dupe || - !strcmp(ptr->directory, service->directory)); - if (dupe) { - log_warn(LD_REND, "Another hidden service is already configured for " - "directory %s.", - rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; - } + goto invalid; + } + + /* XXX This duplicate check has two problems: + * + * a) It's O(n^2), but the same comment from the bottom of + * rend_config_services() should apply. + * + * b) We only compare directory paths as strings, so we can't + * detect two distinct paths that specify the same directory + * (which can arise from symlinks, case-insensitivity, bind + * mounts, etc.). + * + * It also can't detect that two separate Tor instances are trying + * to use the same HiddenServiceDir; for that, we would need a + * lock file. But this is enough to detect a simple mistake that + * at least one person has actually made. + */ + if (!rend_service_is_ephemeral(service)) { + /* Skip dupe for ephemeral services. */ + SMARTLIST_FOREACH(service_list, rend_service_t *, ptr, + dupe = dupe || + !strcmp(ptr->directory, service->directory)); + if (dupe) { + log_warn(LD_REND, "Another hidden service is already configured for " + "directory %s.", + rend_service_escaped_dir(service)); + goto invalid; } - log_debug(LD_REND,"Configuring service with directory %s", - rend_service_escaped_dir(service)); - for (i = 0; i < smartlist_len(service->ports); ++i) { - p = smartlist_get(service->ports, i); - if (!(p->is_unix_addr)) { - log_debug(LD_REND, - "Service maps port %d to %s", - p->virtual_port, - fmt_addrport(&p->real_addr, p->real_port)); - } else { + } + + /* Valid. */ + return 0; + invalid: + return -1; +} + +/** Add it to <b>service_list</b>, or to the global rend_service_list if + * <b>service_list</b> is NULL. Return 0 on success. On failure, free + * <b>service</b> and return -1. Takes ownership of <b>service</b>. */ +static int +rend_add_service(smartlist_t *service_list, rend_service_t *service) +{ + int i; + rend_service_port_config_t *p; + + tor_assert(service); + + smartlist_t *s_list = rend_get_service_list_mutable(service_list); + /* We must have a service list, even if it's a temporary one, so we can + * check for duplicate services */ + if (BUG(!s_list)) { + return -1; + } + + service->intro_nodes = smartlist_new(); + service->expiring_nodes = smartlist_new(); + + log_debug(LD_REND,"Configuring service with directory %s", + rend_service_escaped_dir(service)); + for (i = 0; i < smartlist_len(service->ports); ++i) { + p = smartlist_get(service->ports, i); + if (!(p->is_unix_addr)) { + log_debug(LD_REND, + "Service maps port %d to %s", + p->virtual_port, + fmt_addrport(&p->real_addr, p->real_port)); + } else { #ifdef HAVE_SYS_UN_H - log_debug(LD_REND, - "Service maps port %d to socket at \"%s\"", - p->virtual_port, p->unix_addr); + log_debug(LD_REND, + "Service maps port %d to socket at \"%s\"", + p->virtual_port, p->unix_addr); #else - log_warn(LD_BUG, - "Service maps port %d to an AF_UNIX socket, but we " - "have no AF_UNIX support on this platform. This is " - "probably a bug.", - p->virtual_port); - rend_service_free(service); - return -1; + log_warn(LD_BUG, + "Service maps port %d to an AF_UNIX socket, but we " + "have no AF_UNIX support on this platform. This is " + "probably a bug.", + p->virtual_port); + rend_service_free(service); + return -1; #endif /* defined(HAVE_SYS_UN_H) */ - } } - /* The service passed all the checks */ - tor_assert(s_list); - smartlist_add(s_list, service); - return 0; } - /* NOTREACHED */ + /* The service passed all the checks */ + tor_assert(s_list); + smartlist_add(s_list, service); + return 0; } /** Return a new rend_service_port_config_t with its path set to @@ -539,18 +531,34 @@ rend_service_check_dir_and_add(smartlist_t *service_list, return rend_add_service(s_list, service); } -/* If this is a reload and there were hidden services configured before, - * keep the introduction points that are still needed and close the - * other ones. */ +/* Helper: Actual implementation of the pruning on reload which we've + * decoupled in order to make the unit test workeable without ugly hacks. + * Furthermore, this function does NOT free any memory but will nullify the + * temporary list pointer whatever happens. */ STATIC void -prune_services_on_reload(smartlist_t *old_service_list, - smartlist_t *new_service_list) +rend_service_prune_list_impl_(void) { origin_circuit_t *ocirc = NULL; - smartlist_t *surviving_services = NULL; + smartlist_t *surviving_services, *old_service_list, *new_service_list; - tor_assert(old_service_list); - tor_assert(new_service_list); + /* When pruning our current service list, we must have a staging list that + * contains what we want to check else it's a code flow error. */ + tor_assert(rend_service_staging_list); + + /* We are about to prune the current list of its dead service so set the + * semantic for that list to be the "old" one. */ + old_service_list = rend_service_list; + /* The staging list is now the "new" list so set this semantic. */ + new_service_list = rend_service_staging_list; + /* After this, whatever happens, we'll use our new list. */ + rend_service_list = new_service_list; + /* Finally, nullify the staging list pointer as we don't need it anymore + * and it needs to be NULL before the next reload. */ + rend_service_staging_list = NULL; + /* Nothing to prune if we have no service list so stop right away. */ + if (!old_service_list) { + return; + } /* This contains all _existing_ services that survives the relaod that is * that haven't been removed from the configuration. The difference between @@ -628,6 +636,27 @@ prune_services_on_reload(smartlist_t *old_service_list, smartlist_free(surviving_services); } +/* Try to prune our main service list using the temporary one that we just + * loaded and parsed successfully. The pruning process decides which onion + * services to keep and which to discard after a reload. */ +void +rend_service_prune_list(void) +{ + smartlist_t *old_service_list = rend_service_list; + /* Don't try to prune anything if we have no staging list. */ + if (!rend_service_staging_list) { + return; + } + rend_service_prune_list_impl_(); + if (old_service_list) { + /* Every remaining service in the old list have been removed from the + * configuration so clean them up safely. */ + SMARTLIST_FOREACH(old_service_list, rend_service_t *, s, + rend_service_free(s)); + smartlist_free(old_service_list); + } +} + /** Set up rend_service_list, based on the values of HiddenServiceDir and * HiddenServicePort in <b>options</b>. Return 0 on success and -1 on * failure. (If <b>validate_only</b> is set, parse, warn and return as @@ -639,24 +668,30 @@ rend_config_services(const or_options_t *options, int validate_only) config_line_t *line; rend_service_t *service = NULL; rend_service_port_config_t *portcfg; - smartlist_t *old_service_list = NULL; - smartlist_t *temp_service_list = NULL; int ok = 0; int rv = -1; - /* Use a temporary service list, so that we can check the new services' - * consistency with each other */ - temp_service_list = smartlist_new(); + /* Use the staging service list so that we can check then do the pruning + * process using the main list at the end. */ + if (rend_service_staging_list == NULL) { + rend_service_staging_list = smartlist_new(); + } for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - /* register the service we just finished parsing - * this code registers every service except the last one parsed, - * which is registered below the loop */ - if (rend_service_check_dir_and_add(temp_service_list, options, service, - validate_only) < 0) { - service = NULL; - goto free_and_return; + if (service) { + /* Validate and register the service we just finished parsing this + * code registers every service except the last one parsed, which is + * validated and registered below the loop. */ + if (rend_validate_service(rend_service_staging_list, service) < 0) { + goto free_and_return; + } + if (rend_service_check_dir_and_add(rend_service_staging_list, options, + service, validate_only) < 0) { + /* The above frees the service on error so nullify the pointer. */ + service = NULL; + goto free_and_return; + } } service = tor_malloc_zero(sizeof(rend_service_t)); service->directory = tor_strdup(line->value); @@ -851,14 +886,23 @@ rend_config_services(const or_options_t *options, int validate_only) } } } + /* Validate the last service that we just parsed. */ + if (service && + rend_validate_service(rend_service_staging_list, service) < 0) { + goto free_and_return; + } /* register the final service after we have finished parsing all services * this code only registers the last service, other services are registered * within the loop. It is ok for this service to be NULL, it is ignored. */ - if (rend_service_check_dir_and_add(temp_service_list, options, service, - validate_only) < 0) { + if (rend_service_check_dir_and_add(rend_service_staging_list, options, + service, validate_only) < 0) { + /* Service object is freed on error so nullify pointer. */ service = NULL; goto free_and_return; } + /* The service is in the staging list so nullify pointer to avoid double + * free of this object in case of error because we lost ownership of it at + * this point. */ service = NULL; /* Free the newly added services if validating */ @@ -867,31 +911,19 @@ rend_config_services(const or_options_t *options, int validate_only) goto free_and_return; } - /* Otherwise, use the newly added services as the new service list - * Since we have now replaced the global service list, from this point on we - * must succeed, or die trying. */ - old_service_list = rend_service_list; - rend_service_list = temp_service_list; - temp_service_list = NULL; - - /* If this is a reload and there were hidden services configured before, - * keep the introduction points that are still needed and close the - * other ones. */ - if (old_service_list && !validate_only) { - prune_services_on_reload(old_service_list, rend_service_list); - /* Every remaining service in the old list have been removed from the - * configuration so clean them up safely. */ - SMARTLIST_FOREACH(old_service_list, rend_service_t *, s, - rend_service_free(s)); - smartlist_free(old_service_list); - } + /* This could be a reload of configuration so try to prune the main list + * using the staging one. And we know we are not in validate mode here. + * After this, the main and staging list will point to the right place and + * be in a quiescent usable state. */ + rend_service_prune_list(); return 0; free_and_return: rend_service_free(service); - SMARTLIST_FOREACH(temp_service_list, rend_service_t *, ptr, + SMARTLIST_FOREACH(rend_service_staging_list, rend_service_t *, ptr, rend_service_free(ptr)); - smartlist_free(temp_service_list); + smartlist_free(rend_service_staging_list); + rend_service_staging_list = NULL; return rv; } @@ -1025,6 +1057,38 @@ rend_service_del_ephemeral(const char *service_id) return 0; } +/* There can be 1 second's delay due to second_elapsed_callback, and perhaps + * another few seconds due to blocking calls. */ +#define INTRO_CIRC_RETRY_PERIOD_SLOP 10 + +/** Log information about the intro point creation rate and current intro + * points for service, upgrading the log level from min_severity to warn if + * we have stopped launching new intro point circuits. */ +static void +rend_log_intro_limit(const rend_service_t *service, int min_severity) +{ + int exceeded_limit = (service->n_intro_circuits_launched >= + rend_max_intro_circs_per_period( + service->n_intro_points_wanted)); + int severity = min_severity; + /* We stopped creating circuits */ + if (exceeded_limit) { + severity = LOG_WARN; + } + time_t intro_period_elapsed = time(NULL) - service->intro_period_started; + tor_assert_nonfatal(intro_period_elapsed >= 0); + log_fn(severity, LD_REND, "Hidden service %s %s %d intro points in the last " + "%d seconds. Intro circuit launches are limited to %d per %d " + "seconds.", + service->service_id, + exceeded_limit ? "exceeded launch limit with" : "launched", + service->n_intro_circuits_launched, + (int)intro_period_elapsed, + rend_max_intro_circs_per_period(service->n_intro_points_wanted), + INTRO_CIRC_RETRY_PERIOD); + rend_service_dump_stats(severity); +} + /** Replace the old value of <b>service</b>-\>desc with one that reflects * the other fields in service. */ @@ -1032,7 +1096,6 @@ static void rend_service_update_descriptor(rend_service_t *service) { rend_service_descriptor_t *d; - origin_circuit_t *circ; int i; rend_service_descriptor_free(service->desc); @@ -1053,9 +1116,10 @@ rend_service_update_descriptor(rend_service_t *service) /* This intro point won't be listed in the descriptor... */ intro_svc->listed_in_last_desc = 0; - circ = find_intro_circuit(intro_svc, service->pk_digest); - if (!circ || circ->base_.purpose != CIRCUIT_PURPOSE_S_INTRO) { - /* This intro point's circuit isn't finished yet. Don't list it. */ + /* circuit_established is set in rend_service_intro_established(), and + * checked every second in rend_consider_services_intro_points(), so it's + * safe to use it here */ + if (!intro_svc->circuit_established) { continue; } @@ -1077,6 +1141,26 @@ rend_service_update_descriptor(rend_service_t *service) intro_svc->time_published = time(NULL); } } + + /* Check that we have the right number of intro points */ + unsigned int have_intro = (unsigned int)smartlist_len(d->intro_nodes); + if (have_intro != service->n_intro_points_wanted) { + int severity; + /* Getting less than we wanted or more than we're allowed is serious */ + if (have_intro < service->n_intro_points_wanted || + have_intro > NUM_INTRO_POINTS_MAX) { + severity = LOG_WARN; + } else { + /* Getting more than we wanted is weird, but less of a problem */ + severity = LOG_NOTICE; + } + log_fn(severity, LD_REND, "Hidden service %s wanted %d intro points, but " + "descriptor was updated with %d instead.", + service->service_id, + service->n_intro_points_wanted, have_intro); + /* Now log an informative message about how we might have got here. */ + rend_log_intro_limit(service, severity); + } } /* Allocate and return a string containing the path to file_name in @@ -1234,7 +1318,8 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service, } /* Make sure the directory was created before calling this function. */ - if (BUG(rend_service_check_private_dir_impl(options, service, 0) < 0)) + if (BUG(hs_check_service_private_dir(options->User, service->directory, + service->dir_group_readable, 0) < 0)) return -1; poison_fname = rend_service_sos_poison_path(service); @@ -1384,32 +1469,6 @@ rend_service_derive_key_digests(struct rend_service_t *s) return 0; } -/* Implements the directory check from rend_service_check_private_dir, - * without doing the single onion poison checks. */ -static int -rend_service_check_private_dir_impl(const or_options_t *options, - const rend_service_t *s, - int create) -{ - cpd_check_t check_opts = CPD_NONE; - if (create) { - check_opts |= CPD_CREATE; - } else { - check_opts |= CPD_CHECK_MODE_ONLY; - check_opts |= CPD_CHECK; - } - if (s->dir_group_readable) { - check_opts |= CPD_GROUP_READ; - } - /* Check/create directory */ - if (check_private_dir(s->directory, check_opts, options->User) < 0) { - log_warn(LD_REND, "Checking service directory %s failed.", s->directory); - return -1; - } - - return 0; -} - /** Make sure that the directory for <b>s</b> is private, using the config in * <b>options</b>. * If <b>create</b> is true: @@ -1430,7 +1489,8 @@ rend_service_check_private_dir(const or_options_t *options, } /* Check/create directory */ - if (rend_service_check_private_dir_impl(options, s, create) < 0) { + if (hs_check_service_private_dir(options->User, s->directory, + s->dir_group_readable, create) < 0) { return -1; } @@ -3668,13 +3728,16 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, * request. Lookup is made in rend_service_desc_has_uploaded(). */ rend_data = rend_data_client_create(service_id, desc->desc_id, NULL, REND_NO_AUTH); - directory_initiate_command_routerstatus_rend(hs_dir, - DIR_PURPOSE_UPLOAD_RENDDESC_V2, - ROUTER_PURPOSE_GENERAL, - DIRIND_ANONYMOUS, NULL, - desc->desc_str, - strlen(desc->desc_str), - 0, rend_data, NULL); + directory_request_t *req = + directory_request_new(DIR_PURPOSE_UPLOAD_RENDDESC_V2); + directory_request_set_routerstatus(req, hs_dir); + directory_request_set_indirection(req, DIRIND_ANONYMOUS); + directory_request_set_payload(req, + desc->desc_str, strlen(desc->desc_str)); + directory_request_set_rend_query(req, rend_data); + directory_initiate_request(req); + directory_request_free(req); + rend_data_free(rend_data); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); @@ -4076,8 +4139,12 @@ rend_consider_services_intro_points(void) /* This retry period is important here so we don't stress circuit * creation. */ + if (now > service->intro_period_started + INTRO_CIRC_RETRY_PERIOD) { - /* One period has elapsed; we can try building circuits again. */ + /* One period has elapsed: + * - if we stopped, we can try building circuits again, + * - if we haven't, we reset the circuit creation counts. */ + rend_log_intro_limit(service, LOG_INFO); service->intro_period_started = now; service->n_intro_circuits_launched = 0; } else if (service->n_intro_circuits_launched >= @@ -4085,6 +4152,7 @@ rend_consider_services_intro_points(void) service->n_intro_points_wanted)) { /* We have failed too many times in this period; wait for the next * one before we try to initiate any more connections. */ + rend_log_intro_limit(service, LOG_WARN); continue; } @@ -4137,8 +4205,6 @@ rend_consider_services_intro_points(void) const node_t *node; rend_intro_point_t *intro; router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC; - if (get_options()->AllowInvalid_ & ALLOW_INVALID_INTRODUCTION) - flags |= CRN_ALLOW_INVALID; router_crn_flags_t direct_flags = flags; direct_flags |= CRN_PREF_ADDR; direct_flags |= CRN_DIRECT_CONN; @@ -4530,3 +4596,19 @@ rend_service_non_anonymous_mode_enabled(const or_options_t *options) return options->HiddenServiceNonAnonymousMode ? 1 : 0; } +#ifdef TOR_UNIT_TESTS + +STATIC void +set_rend_service_list(smartlist_t *new_list) +{ + rend_service_list = new_list; +} + +STATIC void +set_rend_rend_service_staging_list(smartlist_t *new_list) +{ + rend_service_staging_list = new_list; +} + +#endif /* TOR_UNIT_TESTS */ + diff --git a/src/or/rendservice.h b/src/or/rendservice.h index 85daaae4e2..1583a6010b 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -133,13 +133,19 @@ STATIC ssize_t encode_establish_intro_cell_legacy(char *cell_body_out, size_t cell_body_out_len, crypto_pk_t *intro_key, char *rend_circ_nonce); -STATIC void prune_services_on_reload(smartlist_t *old_service_list, - smartlist_t *new_service_list); +#ifdef TOR_UNIT_TESTS -#endif +STATIC void set_rend_service_list(smartlist_t *new_list); +STATIC void set_rend_rend_service_staging_list(smartlist_t *new_list); +STATIC void rend_service_prune_list_impl_(void); + +#endif /* TOR_UNIT_TESTS */ + +#endif /* RENDSERVICE_PRIVATE */ int num_rend_services(void); int rend_config_services(const or_options_t *options, int validate_only); +void rend_service_prune_list(void); int rend_service_load_all_keys(const smartlist_t *service_list); void rend_services_add_filenames_to_lists(smartlist_t *open_lst, smartlist_t *stat_lst); diff --git a/src/or/rephist.c b/src/or/rephist.c index 8bcd7396aa..72a5cc5a9b 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -84,9 +84,13 @@ #include "router.h" #include "routerlist.h" #include "ht.h" +#include "channelpadding.h" + +#include "channelpadding.h" +#include "connection_or.h" static void bw_arrays_init(void); -static void predicted_ports_init(void); +static void predicted_ports_alloc(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -165,6 +169,44 @@ typedef struct or_history_t { digestmap_t *link_history_map; } or_history_t; +/** + * This structure holds accounting needed to calculate the padding overhead. + */ +typedef struct padding_counts_t { + /** Total number of cells we have received, including padding */ + uint64_t read_cell_count; + /** Total number of cells we have sent, including padding */ + uint64_t write_cell_count; + /** Total number of CELL_PADDING cells we have received */ + uint64_t read_pad_cell_count; + /** Total number of CELL_PADDING cells we have sent */ + uint64_t write_pad_cell_count; + /** Total number of read cells on padding-enabled conns */ + uint64_t enabled_read_cell_count; + /** Total number of sent cells on padding-enabled conns */ + uint64_t enabled_write_cell_count; + /** Total number of read CELL_PADDING cells on padding-enabled cons */ + uint64_t enabled_read_pad_cell_count; + /** Total number of sent CELL_PADDING cells on padding-enabled cons */ + uint64_t enabled_write_pad_cell_count; + /** Total number of RELAY_DROP cells we have received */ + uint64_t read_drop_cell_count; + /** Total number of RELAY_DROP cells we have sent */ + uint64_t write_drop_cell_count; + /** The maximum number of padding timers we've seen in 24 hours */ + uint64_t maximum_chanpad_timers; + /** When did we first copy padding_current into padding_published? */ + char first_published_at[ISO_TIME_LEN+1]; +} padding_counts_t; + +/** Holds the current values of our padding statistics. + * It is not published until it is transferred to padding_published. */ +static padding_counts_t padding_current; + +/** Remains fixed for a 24 hour period, and then is replaced + * by a redacted copy of padding_current */ +static padding_counts_t padding_published; + /** When did we last multiply all routers' weighted_run_length and * total_run_weights by STABILITY_ALPHA? */ static time_t stability_last_downrated = 0; @@ -264,7 +306,7 @@ rep_hist_init(void) { history_map = digestmap_new(); bw_arrays_init(); - predicted_ports_init(); + predicted_ports_alloc(); } /** Helper: note that we are no longer connected to the router with history @@ -1758,6 +1800,40 @@ typedef struct predicted_port_t { /** A list of port numbers that have been used recently. */ static smartlist_t *predicted_ports_list=NULL; +/** How long do we keep predicting circuits? */ +static int prediction_timeout=0; +/** When was the last time we added a prediction entry (HS or port) */ +static time_t last_prediction_add_time=0; + +/** + * How much time left until we stop predicting circuits? + */ +int +predicted_ports_prediction_time_remaining(time_t now) +{ + time_t idle_delta = now - last_prediction_add_time; + + /* Protect against overflow of return value. This can happen if the clock + * jumps backwards in time. Update the last prediction time (aka last + * active time) to prevent it. This update is preferable to using monotonic + * time because it prevents clock jumps into the past from simply causing + * very long idle timeouts while the monotonic time stands still. */ + if (last_prediction_add_time > now) { + last_prediction_add_time = now; + idle_delta = 0; + } + + /* Protect against underflow of the return value. This can happen for very + * large periods of inactivity/system sleep. */ + if (idle_delta > prediction_timeout) + return 0; + + if (BUG((prediction_timeout - idle_delta) > INT_MAX)) { + return INT_MAX; + } + + return (int)(prediction_timeout - idle_delta); +} /** We just got an application request for a connection with * port <b>port</b>. Remember it for the future, so we can keep @@ -1767,21 +1843,40 @@ static void add_predicted_port(time_t now, uint16_t port) { predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t)); + + // If the list is empty, re-randomize predicted ports lifetime + if (!any_predicted_circuits(now)) { + prediction_timeout = channelpadding_get_circuits_available_timeout(); + } + + last_prediction_add_time = now; + + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ building " + "for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); + pp->port = port; pp->time = now; rephist_total_alloc += sizeof(*pp); smartlist_add(predicted_ports_list, pp); } -/** Initialize whatever memory and structs are needed for predicting +/** + * Allocate whatever memory and structs are needed for predicting * which ports will be used. Also seed it with port 80, so we'll build * circuits on start-up. */ static void -predicted_ports_init(void) +predicted_ports_alloc(void) { predicted_ports_list = smartlist_new(); - add_predicted_port(time(NULL), 80); /* add one to kickstart us */ +} + +void +predicted_ports_init(void) +{ + add_predicted_port(time(NULL), 443); // Add a port to get us started } /** Free whatever memory is needed for predicting which ports will @@ -1812,6 +1907,12 @@ rep_hist_note_used_port(time_t now, uint16_t port) SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { if (pp->port == port) { pp->time = now; + + last_prediction_add_time = now; + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ " + "building for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); return; } } SMARTLIST_FOREACH_END(pp); @@ -1828,7 +1929,8 @@ rep_hist_get_predicted_ports(time_t now) int predicted_circs_relevance_time; smartlist_t *out = smartlist_new(); tor_assert(predicted_ports_list); - predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + + predicted_circs_relevance_time = prediction_timeout; /* clean out obsolete entries */ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { @@ -1888,6 +1990,18 @@ static time_t predicted_internal_capacity_time = 0; void rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity) { + // If the list is empty, re-randomize predicted ports lifetime + if (!any_predicted_circuits(now)) { + prediction_timeout = channelpadding_get_circuits_available_timeout(); + } + + last_prediction_add_time = now; + + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ building " + "for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); + predicted_internal_time = now; if (need_uptime) predicted_internal_uptime_time = now; @@ -1901,7 +2015,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, int *need_capacity) { int predicted_circs_relevance_time; - predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + + predicted_circs_relevance_time = prediction_timeout; if (!predicted_internal_time) { /* initialize it */ predicted_internal_time = now; @@ -1923,7 +2038,7 @@ int any_predicted_circuits(time_t now) { int predicted_circs_relevance_time; - predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + predicted_circs_relevance_time = prediction_timeout; return smartlist_len(predicted_ports_list) || predicted_internal_time + predicted_circs_relevance_time >= now; @@ -3210,8 +3325,7 @@ rep_hist_hs_stats_write(time_t now) return start_of_hs_stats_interval + WRITE_STATS_INTERVAL; } -#define MAX_LINK_PROTO_TO_LOG 4 -static uint64_t link_proto_count[MAX_LINK_PROTO_TO_LOG+1][2]; +static uint64_t link_proto_count[MAX_LINK_PROTO+1][2]; /** Note that we negotiated link protocol version <b>link_proto</b>, on * a connection that started here iff <b>started_here</b> is true. @@ -3220,7 +3334,7 @@ void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here) { started_here = !!started_here; /* force to 0 or 1 */ - if (link_proto > MAX_LINK_PROTO_TO_LOG) { + if (link_proto > MAX_LINK_PROTO) { log_warn(LD_BUG, "Can't log link protocol %u", link_proto); return; } @@ -3228,6 +3342,165 @@ rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here) link_proto_count[link_proto][started_here]++; } +/** + * Update the maximum count of total pending channel padding timers + * in this period. + */ +void +rep_hist_padding_count_timers(uint64_t num_timers) +{ + if (num_timers > padding_current.maximum_chanpad_timers) { + padding_current.maximum_chanpad_timers = num_timers; + } +} + +/** + * Count a cell that we sent for padding overhead statistics. + * + * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be + * counted for PADDING_TYPE_TOTAL. + */ +void +rep_hist_padding_count_write(padding_type_t type) +{ + switch (type) { + case PADDING_TYPE_DROP: + padding_current.write_drop_cell_count++; + break; + case PADDING_TYPE_CELL: + padding_current.write_pad_cell_count++; + break; + case PADDING_TYPE_TOTAL: + padding_current.write_cell_count++; + break; + case PADDING_TYPE_ENABLED_TOTAL: + padding_current.enabled_write_cell_count++; + break; + case PADDING_TYPE_ENABLED_CELL: + padding_current.enabled_write_pad_cell_count++; + break; + } +} + +/** + * Count a cell that we've received for padding overhead statistics. + * + * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be + * counted for PADDING_TYPE_TOTAL. + */ +void +rep_hist_padding_count_read(padding_type_t type) +{ + switch (type) { + case PADDING_TYPE_DROP: + padding_current.read_drop_cell_count++; + break; + case PADDING_TYPE_CELL: + padding_current.read_pad_cell_count++; + break; + case PADDING_TYPE_TOTAL: + padding_current.read_cell_count++; + break; + case PADDING_TYPE_ENABLED_TOTAL: + padding_current.enabled_read_cell_count++; + break; + case PADDING_TYPE_ENABLED_CELL: + padding_current.enabled_read_pad_cell_count++; + break; + } +} + +/** + * Reset our current padding statistics. Called once every 24 hours. + */ +void +rep_hist_reset_padding_counts(void) +{ + memset(&padding_current, 0, sizeof(padding_current)); +} + +/** + * Copy our current cell counts into a structure for listing in our + * extra-info descriptor. Also perform appropriate rounding and redaction. + * + * This function is called once every 24 hours. + */ +#define MIN_CELL_COUNTS_TO_PUBLISH 1 +#define ROUND_CELL_COUNTS_TO 10000 +void +rep_hist_prep_published_padding_counts(time_t now) +{ + memcpy(&padding_published, &padding_current, sizeof(padding_published)); + + if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH || + padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) { + memset(&padding_published, 0, sizeof(padding_published)); + return; + } + + format_iso_time(padding_published.first_published_at, now); +#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \ + ROUND_CELL_COUNTS_TO) + ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_cell_count); + ROUND_AND_SET_COUNT(padding_published.read_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count); +#undef ROUND_AND_SET_COUNT +} + +/** + * Returns an allocated string for extra-info documents for publishing + * padding statistics from the last 24 hour interval. + */ +char * +rep_hist_get_padding_count_lines(void) +{ + char *result = NULL; + + if (!padding_published.read_cell_count || + !padding_published.write_cell_count) { + return NULL; + } + + tor_asprintf(&result, "padding-counts %s (%d s)" + " bin-size="U64_FORMAT + " write-drop="U64_FORMAT + " write-pad="U64_FORMAT + " write-total="U64_FORMAT + " read-drop="U64_FORMAT + " read-pad="U64_FORMAT + " read-total="U64_FORMAT + " enabled-read-pad="U64_FORMAT + " enabled-read-total="U64_FORMAT + " enabled-write-pad="U64_FORMAT + " enabled-write-total="U64_FORMAT + " max-chanpad-timers="U64_FORMAT + "\n", + padding_published.first_published_at, + REPHIST_CELL_PADDING_COUNTS_INTERVAL, + U64_PRINTF_ARG(ROUND_CELL_COUNTS_TO), + U64_PRINTF_ARG(padding_published.write_drop_cell_count), + U64_PRINTF_ARG(padding_published.write_pad_cell_count), + U64_PRINTF_ARG(padding_published.write_cell_count), + U64_PRINTF_ARG(padding_published.read_drop_cell_count), + U64_PRINTF_ARG(padding_published.read_pad_cell_count), + U64_PRINTF_ARG(padding_published.read_cell_count), + U64_PRINTF_ARG(padding_published.enabled_read_pad_cell_count), + U64_PRINTF_ARG(padding_published.enabled_read_cell_count), + U64_PRINTF_ARG(padding_published.enabled_write_pad_cell_count), + U64_PRINTF_ARG(padding_published.enabled_write_cell_count), + U64_PRINTF_ARG(padding_published.maximum_chanpad_timers) + ); + + return result; +} + /** Log a heartbeat message explaining how many connections of each link * protocol version we have used. */ diff --git a/src/or/rephist.h b/src/or/rephist.h index ff4810a56d..2b1c2e7ec7 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -48,6 +48,7 @@ double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); long rep_hist_get_weighted_time_known(const char *id, time_t when); int rep_hist_have_measured_enough_stability(void); +void predicted_ports_init(void); void rep_hist_note_used_port(time_t now, uint16_t port); smartlist_t *rep_hist_get_predicted_ports(time_t now); void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports); @@ -59,6 +60,7 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime, int any_predicted_circuits(time_t now); int rep_hist_circbuilding_dormant(time_t now); +int predicted_ports_prediction_time_remaining(time_t now); void note_crypto_pk_op(pk_op_t operation); void dump_pk_ops(int severity); @@ -119,5 +121,30 @@ extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; #endif +/** + * Represents the type of a cell for padding accounting + */ +typedef enum padding_type_t { + /** A RELAY_DROP cell */ + PADDING_TYPE_DROP, + /** A CELL_PADDING cell */ + PADDING_TYPE_CELL, + /** Total counts of padding and non-padding together */ + PADDING_TYPE_TOTAL, + /** Total cell counts for all padding-enabled channels */ + PADDING_TYPE_ENABLED_TOTAL, + /** CELL_PADDING counts for all padding-enabled channels */ + PADDING_TYPE_ENABLED_CELL +} padding_type_t; + +/** The amount of time over which the padding cell counts were counted */ +#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60) +void rep_hist_padding_count_read(padding_type_t type); +void rep_hist_padding_count_write(padding_type_t type); +char *rep_hist_get_padding_count_lines(void); +void rep_hist_reset_padding_counts(void); +void rep_hist_prep_published_padding_counts(time_t now); +void rep_hist_padding_count_timers(uint64_t num_timers); + #endif diff --git a/src/or/replaycache.c b/src/or/replaycache.c index 8290fa6964..3d42deb90a 100644 --- a/src/or/replaycache.c +++ b/src/or/replaycache.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2012-2016, The Tor Project, Inc. */ + /* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/replaycache.h b/src/or/replaycache.h index 64a6caf5f5..0d637939a4 100644 --- a/src/or/replaycache.h +++ b/src/or/replaycache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/router.c b/src/or/router.c index f6b03cde27..2187a76b48 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTER_PRIVATE @@ -148,6 +148,51 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last) tor_mutex_release(key_lock); } +/** Expire our old set of onion keys. This is done by setting + * last_curve25519_onion_key and lastonionkey to all zero's and NULL + * respectively. + * + * This function does not perform any grace period checks for the old onion + * keys. + */ +void +expire_old_onion_keys(void) +{ + char *fname = NULL; + + tor_mutex_acquire(key_lock); + + /* Free lastonionkey and set it to NULL. */ + if (lastonionkey) { + crypto_pk_free(lastonionkey); + lastonionkey = NULL; + } + + /* We zero out the keypair. See the tor_mem_is_zero() check made in + * construct_ntor_key_map() below. */ + memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); + + tor_mutex_release(key_lock); + + fname = get_datadir_fname2("keys", "secret_onion_key.old"); + if (file_status(fname) == FN_FILE) { + if (tor_unlink(fname) != 0) { + log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s", + fname, strerror(errno)); + } + } + tor_free(fname); + + fname = get_datadir_fname2("keys", "secret_onion_key_ntor.old"); + if (file_status(fname) == FN_FILE) { + if (tor_unlink(fname) != 0) { + log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s", + fname, strerror(errno)); + } + } + tor_free(fname); +} + /** Return the current secret onion key for the ntor handshake. Must only * be called from the main thread. */ static const curve25519_keypair_t * @@ -683,6 +728,47 @@ v3_authority_check_key_expiry(void) last_warned = now; } +/** Get the lifetime of an onion key in days. This value is defined by the + * network consesus parameter "onion-key-rotation-days". Always returns a value + * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and + * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>. + */ +static int +get_onion_key_rotation_days_(void) +{ + return networkstatus_get_param(NULL, + "onion-key-rotation-days", + DEFAULT_ONION_KEY_LIFETIME_DAYS, + MIN_ONION_KEY_LIFETIME_DAYS, + MAX_ONION_KEY_LIFETIME_DAYS); +} + +/** Get the current lifetime of an onion key in seconds. This value is defined + * by the network consesus parameter "onion-key-rotation-days", but the value + * is converted to seconds. + */ +int +get_onion_key_lifetime(void) +{ + return get_onion_key_rotation_days_()*24*60*60; +} + +/** Get the grace period of an onion key in seconds. This value is defined by + * the network consesus parameter "onion-key-grace-period-days", but the value + * is converted to seconds. + */ +int +get_onion_key_grace_period(void) +{ + int grace_period; + grace_period = networkstatus_get_param(NULL, + "onion-key-grace-period-days", + DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS, + MIN_ONION_KEY_GRACE_PERIOD_DAYS, + get_onion_key_rotation_days_()); + return grace_period*24*60*60; +} + /** Set up Tor's TLS contexts, based on our configuration and keys. Return 0 * on success, and -1 on failure. */ int @@ -693,12 +779,6 @@ router_initialize_tls_context(void) int lifetime = options->SSLKeyLifetime; if (public_server_mode(options)) flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER; - if (options->TLSECGroup) { - if (!strcasecmp(options->TLSECGroup, "P256")) - flags |= TOR_TLS_CTX_USE_ECDHE_P256; - else if (!strcasecmp(options->TLSECGroup, "P224")) - flags |= TOR_TLS_CTX_USE_ECDHE_P224; - } if (!lifetime) { /* we should guess a good ssl cert lifetime */ /* choose between 5 and 365 days, and round to the day */ @@ -929,7 +1009,7 @@ init_keys(void) /* We have no LastRotatedOnionKey set; either we just created the key * or it's a holdover from 0.1.2.4-alpha-dev or earlier. In either case, * start the clock ticking now so that we will eventually rotate it even - * if we don't stay up for a full MIN_ONION_KEY_LIFETIME. */ + * if we don't stay up for the full lifetime of an onion key. */ state->LastRotatedOnionKey = onionkey_set_at = now; or_state_mark_dirty(state, options->AvoidDiskWrites ? time(NULL)+3600 : 0); @@ -1385,13 +1465,23 @@ consider_testing_reachability(int test_or, int test_dir) !connection_get_by_type_addr_port_purpose( CONN_TYPE_DIR, &addr, me->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)) { + tor_addr_port_t my_orport, my_dirport; + memcpy(&my_orport.addr, &addr, sizeof(addr)); + memcpy(&my_dirport.addr, &addr, sizeof(addr)); + my_orport.port = me->or_port; + my_dirport.port = me->dir_port; /* ask myself, via tor, for my server descriptor. */ - directory_initiate_command(&addr, me->or_port, - &addr, me->dir_port, - me->cache_info.identity_digest, - DIR_PURPOSE_FETCH_SERVERDESC, - ROUTER_PURPOSE_GENERAL, - DIRIND_ANON_DIRPORT, "authority.z", NULL, 0, 0); + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); + directory_request_set_or_addr_port(req, &my_orport); + directory_request_set_dir_addr_port(req, &my_dirport); + directory_request_set_directory_id_digest(req, + me->cache_info.identity_digest); + // ask via an anon circuit, connecting to our dirport. + directory_request_set_indirection(req, DIRIND_ANON_DIRPORT); + directory_request_set_resource(req, "authority.z"); + directory_initiate_request(req); + directory_request_free(req); } } @@ -1568,8 +1658,7 @@ MOCK_IMPL(int, server_mode,(const or_options_t *options)) { if (options->ClientOnly) return 0; - /* XXXX I believe we can kill off ORListenAddress here.*/ - return (options->ORPort_set || options->ORListenAddress); + return (options->ORPort_set); } /** Return true iff we are trying to be a non-bridge server. @@ -2193,17 +2282,15 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) } if (options->MyFamily && ! options->BridgeRelay) { - smartlist_t *family; if (!warned_nonexistent_family) warned_nonexistent_family = smartlist_new(); - family = smartlist_new(); ri->declared_family = smartlist_new(); - smartlist_split_string(family, options->MyFamily, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); - SMARTLIST_FOREACH_BEGIN(family, char *, name) { + config_line_t *family; + for (family = options->MyFamily; family; family = family->next) { + char *name = family->value; const node_t *member; if (!strcasecmp(name, options->Nickname)) - goto skip; /* Don't list ourself, that's redundant */ + continue; /* Don't list ourself, that's redundant */ else member = node_get_by_nickname(name, 1); if (!member) { @@ -2222,8 +2309,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) smartlist_add_strdup(warned_nonexistent_family, name); } if (is_legal) { - smartlist_add(ri->declared_family, name); - name = NULL; + smartlist_add_strdup(ri->declared_family, name); } } else if (router_digest_is_me(member->identity)) { /* Don't list ourself in our own family; that's redundant */ @@ -2237,15 +2323,11 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (smartlist_contains_string(warned_nonexistent_family, name)) smartlist_string_remove(warned_nonexistent_family, name); } - skip: - tor_free(name); - } SMARTLIST_FOREACH_END(name); + } /* remove duplicates from the list */ smartlist_sort_strings(ri->declared_family); smartlist_uniq_strings(ri->declared_family); - - smartlist_free(family); } /* Now generate the extrainfo. */ @@ -2761,7 +2843,7 @@ router_dump_router_to_string(routerinfo_t *router, make_ntor_onion_key_crosscert(ntor_keypair, &router->cache_info.signing_key_cert->signing_key, router->cache_info.published_on, - MIN_ONION_KEY_LIFETIME, &sign); + get_onion_key_lifetime(), &sign); if (!cert) { log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!"); goto err; @@ -2847,7 +2929,7 @@ router_dump_router_to_string(routerinfo_t *router, "onion-key\n%s" "signing-key\n%s" "%s%s" - "%s%s%s%s", + "%s%s%s", router->nickname, address, router->or_port, @@ -2870,8 +2952,7 @@ router_dump_router_to_string(routerinfo_t *router, ntor_cc_line ? ntor_cc_line : "", family_line, we_are_hibernating() ? "hibernating 1\n" : "", - "hidden-service-dir\n", - options->AllowSingleHopExits ? "allow-single-hop-exits\n" : ""); + "hidden-service-dir\n"); if (options->ContactInfo && strlen(options->ContactInfo)) { const char *ci = options->ContactInfo; @@ -3200,6 +3281,12 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, } } + if (options->PaddingStatistics) { + contents = rep_hist_get_padding_count_lines(); + if (contents) + smartlist_add(chunks, contents); + } + /* Add information about the pluggable transports we support. */ if (options->ServerTransportPlugin) { char *pluggable_transports = pt_get_extra_info_descriptor_string(); diff --git a/src/or/router.h b/src/or/router.h index c30a0301b7..9c5def5218 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,10 +27,13 @@ crypto_pk_t *get_my_v3_authority_signing_key(void); authority_cert_t *get_my_v3_legacy_cert(void); crypto_pk_t *get_my_v3_legacy_signing_key(void); void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last); +void expire_old_onion_keys(void); void rotate_onion_key(void); crypto_pk_t *init_key_from_file(const char *fname, int generate, int severity, int log_greeting); void v3_authority_check_key_expiry(void); +int get_onion_key_lifetime(void); +int get_onion_key_grace_period(void); di_digest256_map_t *construct_ntor_key_map(void); void ntor_key_map_free(di_digest256_map_t *map); diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c index aa7aee4b02..71889d2721 100644 --- a/src/or/routerkeys.c +++ b/src/or/routerkeys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -690,6 +690,10 @@ load_ed_keys(const or_options_t *options, time_t now) tor_cert_t *auth_cert = NULL; int signing_key_changed = 0; + // It is later than 1972, since otherwise there would be no C compilers. + // (Try to diagnose #22466.) + tor_assert_nonfatal(now >= 2 * 365 * 86400); + #define FAIL(msg) do { \ log_warn(LD_OR, (msg)); \ goto err; \ diff --git a/src/or/routerkeys.h b/src/or/routerkeys.h index 845abb4c70..c10cf32a71 100644 --- a/src/or/routerkeys.h +++ b/src/or/routerkeys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ROUTERKEYS_H diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 45b3838792..0e45f63f70 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -426,8 +426,8 @@ list_sk_digests_for_authority_id, (const char *digest)) * download_status_t or NULL if none exists. */ MOCK_IMPL(download_status_t *, - download_status_for_authority_id_and_sk, - (const char *id_digest, const char *sk_digest)) +download_status_for_authority_id_and_sk,(const char *id_digest, + const char *sk_digest)) { download_status_t *dl = NULL; cert_list_t *cl = NULL; @@ -947,6 +947,7 @@ authority_certs_fetch_resource_impl(const char *resource, const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; + directory_request_t *req = NULL; /* If we've just downloaded a consensus from a bridge, re-use that * bridge */ if (options->UseBridges && node && node->ri && !get_via_tor) { @@ -955,23 +956,25 @@ authority_certs_fetch_resource_impl(const char *resource, /* we are willing to use a non-preferred address if we need to */ fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &or_ap); - directory_initiate_command(&or_ap.addr, or_ap.port, - NULL, 0, /*no dirport*/ - dir_hint, - DIR_PURPOSE_FETCH_CERTIFICATE, - 0, - indirection, - resource, NULL, 0, 0); - return; - } - if (rs) { - /* If we've just downloaded a consensus from a directory, re-use that + req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); + directory_request_set_or_addr_port(req, &or_ap); + if (dir_hint) + directory_request_set_directory_id_digest(req, dir_hint); + } else if (rs) { + /* And if we've just downloaded a consensus from a directory, re-use that * directory */ - directory_initiate_command_routerstatus(rs, - DIR_PURPOSE_FETCH_CERTIFICATE, - 0, indirection, resource, NULL, - 0, 0, NULL); + req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); + directory_request_set_routerstatus(req, rs); + } + + if (req) { + /* We've set up a request object -- fill in the other request fields, and + * send the request. */ + directory_request_set_indirection(req, indirection); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); return; } @@ -2317,17 +2320,16 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router) * we can pick a node for a circuit. */ void -router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, - int need_uptime, int need_capacity, - int need_guard, int need_desc, - int pref_addr, int direct_conn) +router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime, + int need_capacity, int need_guard, + int need_desc, int pref_addr, + int direct_conn) { const int check_reach = !router_skip_or_reachability(get_options(), pref_addr); /* XXXX MOVE */ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { - if (!node->is_running || - (!node->is_valid && !allow_invalid)) + if (!node->is_running || !node->is_valid) continue; if (need_desc && !(node->ri || (node->rs && node->md))) continue; @@ -2773,8 +2775,6 @@ node_sl_choose_by_bandwidth(const smartlist_t *sl, * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the * advertised capacity of each router. - * If <b>CRN_ALLOW_INVALID</b> is not set in flags, consider only Valid - * routers. * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers. * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if * picking an exit node, otherwise we weight bandwidths for picking a relay @@ -2795,7 +2795,6 @@ router_choose_random_node(smartlist_t *excludedsmartlist, const int need_uptime = (flags & CRN_NEED_UPTIME) != 0; const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; - const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; const int need_desc = (flags & CRN_NEED_DESC) != 0; const int pref_addr = (flags & CRN_PREF_ADDR) != 0; @@ -2811,20 +2810,17 @@ router_choose_random_node(smartlist_t *excludedsmartlist, rule = weight_for_exit ? WEIGHT_FOR_EXIT : (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); - /* Exclude relays that allow single hop exit circuits, if the user - * wants to (such relays might be risky) */ - if (get_options()->ExcludeSingleHopRelays) { - SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node, - if (node_allows_single_hop_exits(node)) { - smartlist_add(excludednodes, node); - }); - } + /* Exclude relays that allow single hop exit circuits. This is an obsolete + * option since 0.2.9.2-alpha and done by default in 0.3.1.0-alpha. */ + SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node, + if (node_allows_single_hop_exits(node)) { + smartlist_add(excludednodes, node); + }); if ((r = routerlist_find_my_routerinfo())) routerlist_add_node_and_family(excludednodes, r); - router_add_running_nodes_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, + router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity, need_guard, need_desc, pref_addr, direct_conn); log_debug(LD_CIRC, @@ -3045,8 +3041,8 @@ router_get_by_extrainfo_digest,(const char *digest)) /** Return the signed descriptor for the extrainfo_t in our routerlist whose * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info * document is known. */ -signed_descriptor_t * -extrainfo_get_by_descriptor_digest(const char *digest) +MOCK_IMPL(signed_descriptor_t *, +extrainfo_get_by_descriptor_digest,(const char *digest)) { extrainfo_t *ei; tor_assert(digest); @@ -4878,9 +4874,10 @@ list_pending_fpsk_downloads(fp_pair_map_t *result) * range.) If <b>source</b> is given, download from <b>source</b>; * otherwise, download from an appropriate random directory server. */ -MOCK_IMPL(STATIC void, initiate_descriptor_downloads, - (const routerstatus_t *source, int purpose, smartlist_t *digests, - int lo, int hi, int pds_flags)) +MOCK_IMPL(STATIC void, +initiate_descriptor_downloads,(const routerstatus_t *source, + int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)) { char *resource, *cp; int digest_len, enc_digest_len; @@ -4932,10 +4929,11 @@ MOCK_IMPL(STATIC void, initiate_descriptor_downloads, if (source) { /* We know which authority or directory mirror we want. */ - directory_initiate_command_routerstatus(source, purpose, - ROUTER_PURPOSE_GENERAL, - DIRIND_ONEHOP, - resource, NULL, 0, 0, NULL); + directory_request_t *req = directory_request_new(purpose); + directory_request_set_routerstatus(req, source); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); } else { directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource, pds_flags, DL_WANT_ANY_DIRSERVER); @@ -5145,7 +5143,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote, continue; /* We would throw it out immediately. */ } if (!we_want_to_fetch_flavor(options, consensus->flavor) && - !client_would_use_router(rs, now, options)) { + !client_would_use_router(rs, now)) { ++n_wouldnt_use; continue; /* We would never use it ourself. */ } diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 8b68d69f28..e0ed4e623a 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -62,10 +62,10 @@ int router_skip_or_reachability(const or_options_t *options, int try_ip_pref); int router_get_my_share_of_directory_requests(double *v3_share_out); void router_reset_status_download_failures(void); int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2); -void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, - int need_uptime, int need_capacity, - int need_guard, int need_desc, - int pref_addr, int direct_conn); +void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime, + int need_capacity, int need_guard, + int need_desc, int pref_addr, + int direct_conn); const routerinfo_t *routerlist_find_my_routerinfo(void); uint32_t router_get_advertised_bandwidth(const routerinfo_t *router); @@ -92,7 +92,8 @@ routerinfo_t *router_get_mutable_by_digest(const char *digest); signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest, (const char *digest)); -signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); +MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest, + (const char *digest)); const char *signed_descriptor_get_body(const signed_descriptor_t *desc); const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc); routerlist_t *router_get_routerlist(void); @@ -123,7 +124,7 @@ static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s); */ static inline int WRA_WAS_ADDED(was_router_added_t s) { - return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; + return s == ROUTER_ADDED_SUCCESSFULLY; } /** Return true iff the outcome code in <b>s</b> indicates that the descriptor * was not added because it was either: diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 98167d44f8..0f6113ccfc 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -359,6 +359,7 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, const char *end_str, char end_c, + int log_severity, const char **start_out, const char **end_out); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, @@ -988,6 +989,41 @@ router_get_router_hash(const char *s, size_t s_len, char *digest) DIGEST_SHA1); } +/** Try to find the start and end of the signed portion of a networkstatus + * document in <b>s</b>. On success, set <b>start_out</b> to the first + * character of the document, and <b>end_out</b> to a position one after the + * final character of the signed document, and return 0. On failure, return + * -1. */ +int +router_get_networkstatus_v3_signed_boundaries(const char *s, + const char **start_out, + const char **end_out) +{ + return router_get_hash_impl_helper(s, strlen(s), + "network-status-version", + "\ndirectory-signature", + ' ', LOG_INFO, + start_out, end_out); +} + +/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the + * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no + * signed portion can be identified. Return 0 on success, -1 on failure. */ +int +router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, + const char *s) +{ + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) { + start = s; + end = s + strlen(s); + } + tor_assert(start); + tor_assert(end); + return crypto_digest256((char*)digest_out, start, end-start, + DIGEST_SHA3_256); +} + /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int @@ -1787,7 +1823,8 @@ router_parse_entry_from_string(const char *s, const char *end, if (router_get_hash_impl_helper(s, end-s, "router ", "\nrouter-sig-ed25519", - ' ', &signed_start, &signed_end) < 0) { + ' ', LOG_WARN, + &signed_start, &signed_end) < 0) { log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor"); goto err; } @@ -2030,6 +2067,9 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, * parse that's covered by the hash. */ int can_dl_again = 0; + if (BUG(s == NULL)) + return NULL; + if (!end) { end = s + strlen(s); } @@ -2137,7 +2177,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (router_get_hash_impl_helper(s, end-s, "extra-info ", "\nrouter-sig-ed25519", - ' ', &signed_start, &signed_end) < 0) { + ' ', LOG_WARN, + &signed_start, &signed_end) < 0) { log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo"); goto err; } @@ -2544,7 +2585,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } } else if (flav == FLAV_MICRODESC) { - offset = -1; /* There is no identity digest */ + offset = -1; /* There is no descriptor digest in an md consensus r line */ } if (vote_rs) { @@ -2664,6 +2705,10 @@ routerstatus_parse_entry_from_string(memarea_t *area, protocol_list_supports_protocol(tok->args[0], PRT_RELAY, 2); rs->supports_ed25519_link_handshake = protocol_list_supports_protocol(tok->args[0], PRT_LINKAUTH, 3); + rs->supports_ed25519_hs_intro = + protocol_list_supports_protocol(tok->args[0], PRT_HSINTRO, 4); + rs->supports_v3_hsdir = + protocol_list_supports_protocol(tok->args[0], PRT_HSDIR, 2); } if ((tok = find_opt_by_keyword(tokens, K_V))) { tor_assert(tok->n_args == 1); @@ -3339,6 +3384,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; common_digests_t ns_digests; + uint8_t sha3_as_signed[DIGEST256_LEN]; const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; struct in_addr in; @@ -3352,7 +3398,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests) || + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -3369,6 +3416,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns = tor_malloc_zero(sizeof(networkstatus_t)); memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed)); tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); tor_assert(tok); @@ -4464,16 +4512,18 @@ static int router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, const char *end_str, char end_c, + int log_severity, const char **start_out, const char **end_out) { const char *start, *end; start = tor_memstr(s, s_len, start_str); if (!start) { - log_warn(LD_DIR,"couldn't find start of hashed material \"%s\"",start_str); + log_fn(log_severity,LD_DIR, + "couldn't find start of hashed material \"%s\"",start_str); return -1; } if (start != s && *(start-1) != '\n') { - log_warn(LD_DIR, + log_fn(log_severity,LD_DIR, "first occurrence of \"%s\" is not at the start of a line", start_str); return -1; @@ -4481,12 +4531,14 @@ router_get_hash_impl_helper(const char *s, size_t s_len, end = tor_memstr(start+strlen(start_str), s_len - (start-s) - strlen(start_str), end_str); if (!end) { - log_warn(LD_DIR,"couldn't find end of hashed material \"%s\"",end_str); + log_fn(log_severity,LD_DIR, + "couldn't find end of hashed material \"%s\"",end_str); return -1; } end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str)); if (!end) { - log_warn(LD_DIR,"couldn't find EOL"); + log_fn(log_severity,LD_DIR, + "couldn't find EOL"); return -1; } ++end; @@ -4510,7 +4562,7 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, digest_algorithm_t alg) { const char *start=NULL, *end=NULL; - if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN, &start,&end)<0) return -1; @@ -4547,7 +4599,7 @@ router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests, const char *end_str, char end_c) { const char *start=NULL, *end=NULL; - if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN, &start,&end)<0) return -1; @@ -4894,6 +4946,8 @@ tor_version_parse(const char *s, tor_version_t *out) #define NUMBER(m) \ do { \ + if (!cp || *cp < '0' || *cp > '9') \ + return -1; \ out->m = (int)tor_parse_uint64(cp, 10, 0, INT32_MAX, &ok, &eos); \ if (!ok) \ return -1; \ diff --git a/src/or/routerparse.h b/src/or/routerparse.h index 648f29b0d3..088f773c5e 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,6 +16,11 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests); +int router_get_networkstatus_v3_signed_boundaries(const char *s, + const char **start_out, + const char **end_out); +int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, + const char *s); int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest); #define DIROBJ_MAX_SIG_LEN 256 char *router_get_dirobj_signature(const char *digest, diff --git a/src/or/routerset.c b/src/or/routerset.c index d0df0a74e6..4906c6a51d 100644 --- a/src/or/routerset.c +++ b/src/or/routerset.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/routerset.h b/src/or/routerset.h index 2e3b4b0fe0..a63677b471 100644 --- a/src/or/routerset.h +++ b/src/or/routerset.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/scheduler.c b/src/or/scheduler.c index 033e6d119c..fac545fba7 100644 --- a/src/or/scheduler.c +++ b/src/or/scheduler.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/or/scheduler.h b/src/or/scheduler.h index 3dcfd2faca..e29c13de7e 100644 --- a/src/or/scheduler.h +++ b/src/or/scheduler.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/shared_random.c b/src/or/shared_random.c index f798a51a9f..25ca0611cd 100644 --- a/src/or/shared_random.c +++ b/src/or/shared_random.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -230,9 +230,7 @@ commit_decode(const char *encoded, sr_commit_t *commit) { int decoded_len = 0; size_t offset = 0; - /* XXX: Needs two extra bytes for the base64 decode calculation matches - * the binary length once decoded. #17868. */ - char b64_decoded[SR_COMMIT_LEN + 2]; + char b64_decoded[SR_COMMIT_LEN]; tor_assert(encoded); tor_assert(commit); @@ -284,9 +282,7 @@ STATIC int reveal_decode(const char *encoded, sr_commit_t *commit) { int decoded_len = 0; - /* XXX: Needs two extra bytes for the base64 decode calculation matches - * the binary length once decoded. #17868. */ - char b64_decoded[SR_REVEAL_LEN + 2]; + char b64_decoded[SR_REVEAL_LEN]; tor_assert(encoded); tor_assert(commit); diff --git a/src/or/shared_random.h b/src/or/shared_random.h index dbb8effeaa..1f027c70e0 100644 --- a/src/or/shared_random.h +++ b/src/or/shared_random.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SHARED_RANDOM_H @@ -36,17 +36,14 @@ /* Length of base64 encoded commit NOT including the NUL terminated byte. * Formula is taken from base64_encode_size. This adds up to 56 bytes. */ -#define SR_COMMIT_BASE64_LEN \ - (((SR_COMMIT_LEN - 1) / 3) * 4 + 4) +#define SR_COMMIT_BASE64_LEN (BASE64_LEN(SR_COMMIT_LEN)) /* Length of base64 encoded reveal NOT including the NUL terminated byte. * Formula is taken from base64_encode_size. This adds up to 56 bytes. */ -#define SR_REVEAL_BASE64_LEN \ - (((SR_REVEAL_LEN - 1) / 3) * 4 + 4) +#define SR_REVEAL_BASE64_LEN (BASE64_LEN(SR_REVEAL_LEN)) /* Length of base64 encoded shared random value. It's 32 bytes long so 44 * bytes from the base64_encode_size formula. That includes the '=' * character at the end. */ -#define SR_SRV_VALUE_BASE64_LEN \ - (((DIGEST256_LEN - 1) / 3) * 4 + 4) +#define SR_SRV_VALUE_BASE64_LEN (BASE64_LEN(DIGEST256_LEN)) /* Assert if commit valid flag is not set. */ #define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid) diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c index 87db9031ee..89d2e8d7f6 100644 --- a/src/or/shared_random_state.c +++ b/src/or/shared_random_state.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/shared_random_state.h b/src/or/shared_random_state.h index 43a7f1d284..3526ad47d3 100644 --- a/src/or/shared_random_state.h +++ b/src/or/shared_random_state.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SHARED_RANDOM_STATE_H diff --git a/src/or/statefile.c b/src/or/statefile.c index a95ba8533c..d0606b3012 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/statefile.h b/src/or/statefile.h index b13743481d..10c09324bc 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATEFILE_H diff --git a/src/or/status.c b/src/or/status.c index fce6a10157..f7be41e412 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/status.h b/src/or/status.h index b97e835037..c1a0033ce0 100644 --- a/src/or/status.h +++ b/src/or/status.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATUS_H diff --git a/src/or/tor_main.c b/src/or/tor_main.c index d67eda2ac9..a3a8838602 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ extern const char tor_git_revision[]; diff --git a/src/or/torcert.c b/src/or/torcert.c index c58f3da2d3..658e620ca5 100644 --- a/src/or/torcert.c +++ b/src/or/torcert.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -302,6 +302,10 @@ tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key, time_t expires, uint8_t **cert) { + // It is later than 1985, since otherwise there would be no C89 + // compilers. (Try to diagnose #22466.) + tor_assert_nonfatal(expires >= 15 * 365 * 86400); + uint8_t *res; rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new(); diff --git a/src/or/torcert.h b/src/or/torcert.h index 090f6b5811..51f7665f1e 100644 --- a/src/or/torcert.h +++ b/src/or/torcert.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TORCERT_H_INCLUDED diff --git a/src/or/transports.c b/src/or/transports.c index 535393b1a1..31849a8d15 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/transports.h b/src/or/transports.h index 7de90dcbec..44a9626e50 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** |