diff options
Diffstat (limited to 'src/or')
54 files changed, 5642 insertions, 1678 deletions
diff --git a/src/or/Makefile.nmake b/src/or/Makefile.nmake index 2ac98cd372..429ae67858 100644 --- a/src/or/Makefile.nmake +++ b/src/or/Makefile.nmake @@ -14,6 +14,7 @@ LIBTOR_OBJECTS = \ addressmap.obj \ buffers.obj \ channel.obj \ + channelpadding.obj \ channeltls.obj \ circpathbias.obj \ circuitbuild.obj \ diff --git a/src/or/bridges.c b/src/or/bridges.c index 255e0a4f7c..ef0638c0a7 100644 --- a/src/or/bridges.c +++ b/src/or/bridges.c @@ -570,12 +570,18 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } - directory_initiate_command(&bridge->addr, bridge->port, - NULL, 0, /*no dirport*/ - bridge->identity, - DIR_PURPOSE_FETCH_SERVERDESC, - ROUTER_PURPOSE_BRIDGE, - DIRIND_ONEHOP, "authority.z", NULL, 0, 0); + tor_addr_port_t bridge_addrport; + memcpy(&bridge_addrport.addr, &bridge->addr, sizeof(tor_addr_t)); + bridge_addrport.port = bridge->port; + + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); + directory_request_set_or_addr_port(req, &bridge_addrport); + directory_request_set_directory_id_digest(req, bridge->identity); + directory_request_set_router_purpose(req, ROUTER_PURPOSE_BRIDGE); + directory_request_set_resource(req, "authority.z"); + directory_initiate_request(req); + directory_request_free(req); } /** Fetching the bridge descriptor from the bridge authority returned a diff --git a/src/or/buffers.c b/src/or/buffers.c index e559f80a1e..58cfdeee84 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1319,7 +1319,7 @@ fetch_from_buf_http(buf_t *buf, /** * Wait this many seconds before warning the user about using SOCKS unsafely - * again (requires that WarnUnsafeSocks is turned on). */ + * again. */ #define SOCKS_WARN_INTERVAL 5 /** Warn that the user application has made an unsafe socks request using @@ -1331,9 +1331,6 @@ log_unsafe_socks_warning(int socks_protocol, const char *address, { static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL); - const or_options_t *options = get_options(); - if (! options->WarnUnsafeSocks) - return; if (safe_socks) { log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP, "Your application (using socks%d to port %d) is giving " @@ -2088,13 +2085,13 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) } /** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the - * zlib state <b>state</b>, appending the result to <b>buf</b>. If + * compression state <b>state</b>, appending the result to <b>buf</b>. If * <b>done</b> is true, flush the data in the state and finish the * compression/uncompression. Return -1 on failure, 0 on success. */ int -write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, - int done) +write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + int done) { char *next; size_t old_avail, avail; @@ -2108,20 +2105,22 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, } next = CHUNK_WRITE_PTR(buf->tail); avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); - switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) { - case TOR_ZLIB_DONE: + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: over = 1; break; - case TOR_ZLIB_ERR: + case TOR_COMPRESS_ERROR: return -1; - case TOR_ZLIB_OK: + case TOR_COMPRESS_OK: if (data_len == 0) over = 1; break; - case TOR_ZLIB_BUF_FULL: + case TOR_COMPRESS_BUFFER_FULL: if (avail) { - /* Zlib says we need more room (ZLIB_BUF_FULL). Start a new chunk - * automatically, whether were going to or not. */ + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ need_new_chunk = 1; } break; diff --git a/src/or/buffers.h b/src/or/buffers.h index c6a5ffaad5..23b58a571a 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -36,8 +36,8 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, int done); +int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, int done); int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); int fetch_from_buf(char *string, size_t string_len, buf_t *buf); int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); diff --git a/src/or/channel.c b/src/or/channel.c index e79fc0760b..df6d7d3423 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -49,6 +49,7 @@ #include "or.h" #include "channel.h" #include "channeltls.h" +#include "channelpadding.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -63,6 +64,7 @@ #include "router.h" #include "routerlist.h" #include "scheduler.h" +#include "compat_time.h" /* Global lists of channels */ @@ -84,6 +86,28 @@ static smartlist_t *active_listeners = NULL; /* All channel_listener_t instances in LISTENING state */ static smartlist_t *finished_listeners = NULL; +/** Map from channel->global_identifier to channel. Contains the same + * elements as all_channels. */ +static HT_HEAD(channel_gid_map, channel_s) channel_gid_map = HT_INITIALIZER(); + +static unsigned +channel_id_hash(const channel_t *chan) +{ + return (unsigned) chan->global_identifier; +} +static int +channel_id_eq(const channel_t *a, const channel_t *b) +{ + return a->global_identifier == b->global_identifier; +} +HT_PROTOTYPE(channel_gid_map, channel_s, gidmap_node, + channel_id_hash, channel_id_eq) +HT_GENERATE2(channel_gid_map, channel_s, gidmap_node, + channel_id_hash, channel_id_eq, + 0.6, tor_reallocarray_, tor_free_) + +HANDLE_IMPL(channel, channel_s,) + /* Counter for ID numbers */ static uint64_t n_channels_allocated = 0; /* @@ -429,6 +453,7 @@ void channel_register(channel_t *chan) { tor_assert(chan); + tor_assert(chan->global_identifier); /* No-op if already registered */ if (chan->registered) return; @@ -443,6 +468,8 @@ channel_register(channel_t *chan) /* Make sure we have all_channels, then add it */ if (!all_channels) all_channels = smartlist_new(); smartlist_add(all_channels, chan); + channel_t *oldval = HT_REPLACE(channel_gid_map, &channel_gid_map, chan); + tor_assert(! oldval); /* Is it finished? */ if (CHANNEL_FINISHED(chan)) { @@ -498,7 +525,9 @@ channel_unregister(channel_t *chan) } /* Get it out of all_channels */ - if (all_channels) smartlist_remove(all_channels, chan); + if (all_channels) smartlist_remove(all_channels, chan); + channel_t *oldval = HT_REMOVE(channel_gid_map, &channel_gid_map, chan); + tor_assert(oldval == NULL || oldval == chan); /* Mark it as unregistered */ chan->registered = 0; @@ -533,7 +562,7 @@ channel_listener_register(channel_listener_t *chan_l) channel_listener_state_to_string(chan_l->state), chan_l->state); - /* Make sure we have all_channels, then add it */ + /* Make sure we have all_listeners, then add it */ if (!all_listeners) all_listeners = smartlist_new(); smartlist_add(all_listeners, chan_l); @@ -578,7 +607,7 @@ channel_listener_unregister(channel_listener_t *chan_l) if (active_listeners) smartlist_remove(active_listeners, chan_l); } - /* Get it out of all_channels */ + /* Get it out of all_listeners */ if (all_listeners) smartlist_remove(all_listeners, chan_l); /* Mark it as unregistered */ @@ -719,15 +748,13 @@ channel_remove_from_digest_map(channel_t *chan) channel_t * channel_find_by_global_id(uint64_t global_identifier) { + channel_t lookup; channel_t *rv = NULL; - if (all_channels && smartlist_len(all_channels) > 0) { - SMARTLIST_FOREACH_BEGIN(all_channels, channel_t *, curr) { - if (curr->global_identifier == global_identifier) { - rv = curr; - break; - } - } SMARTLIST_FOREACH_END(curr); + lookup.global_identifier = global_identifier; + rv = HT_FIND(channel_gid_map, &channel_gid_map, &lookup); + if (rv) { + tor_assert(rv->global_identifier == global_identifier); } return rv; @@ -809,6 +836,83 @@ channel_next_with_rsa_identity(channel_t *chan) } /** + * Relays run this once an hour to look over our list of channels to other + * relays. It prints out some statistics if there are multiple connections + * to many relays. + * + * This function is similar to connection_or_set_bad_connections(), + * and probably could be adapted to replace it, if it was modified to actually + * take action on any of these connections. + */ +void +channel_check_for_duplicates(void) +{ + channel_idmap_entry_t **iter; + channel_t *chan; + int total_relay_connections = 0, total_relays = 0, total_canonical = 0; + int total_half_canonical = 0; + int total_gt_one_connection = 0, total_gt_two_connections = 0; + int total_gt_four_connections = 0; + + HT_FOREACH(iter, channel_idmap, &channel_identity_map) { + int connections_to_relay = 0; + + /* Only consider relay connections */ + if (!connection_or_digest_is_known_relay((char*)(*iter)->digest)) + continue; + + total_relays++; + + for (chan = TOR_LIST_FIRST(&(*iter)->channel_list); chan; + chan = channel_next_with_rsa_identity(chan)) { + + if (CHANNEL_CONDEMNED(chan) || !CHANNEL_IS_OPEN(chan)) + continue; + + connections_to_relay++; + total_relay_connections++; + + if (chan->is_canonical(chan, 0)) total_canonical++; + + if (!chan->is_canonical_to_peer && chan->is_canonical(chan, 0) + && chan->is_canonical(chan, 1)) { + total_half_canonical++; + } + } + + if (connections_to_relay > 1) total_gt_one_connection++; + if (connections_to_relay > 2) total_gt_two_connections++; + if (connections_to_relay > 4) total_gt_four_connections++; + } + +#define MIN_RELAY_CONNECTIONS_TO_WARN 5 + + /* If we average 1.5 or more connections per relay, something is wrong */ + if (total_relays > MIN_RELAY_CONNECTIONS_TO_WARN && + total_relay_connections >= 1.5*total_relays) { + log_notice(LD_OR, + "Your relay has a very large number of connections to other relays. " + "Is your outbound address the same as your relay address? " + "Found %d connections to %d relays. Found %d current canonical " + "connections, in %d of which we were a non-canonical peer. " + "%d relays had more than 1 connection, %d had more than 2, and " + "%d had more than 4 connections.", + total_relay_connections, total_relays, total_canonical, + total_half_canonical, total_gt_one_connection, + total_gt_two_connections, total_gt_four_connections); + } else { + log_info(LD_OR, "Performed connection pruning. " + "Found %d connections to %d relays. Found %d current canonical " + "connections, in %d of which we were a non-canonical peer. " + "%d relays had more than 1 connection, %d had more than 2, and " + "%d had more than 4 connections.", + total_relay_connections, total_relays, total_canonical, + total_half_canonical, total_gt_one_connection, + total_gt_two_connections, total_gt_four_connections); + } +} + +/** * Initialize a channel * * This function should be called by subclasses to set up some per-channel @@ -822,7 +926,7 @@ channel_init(channel_t *chan) tor_assert(chan); /* Assign an ID and bump the counter */ - chan->global_identifier = n_channels_allocated++; + chan->global_identifier = ++n_channels_allocated; /* Init timestamp */ chan->timestamp_last_had_circuits = time(NULL); @@ -861,7 +965,7 @@ channel_init_listener(channel_listener_t *chan_l) tor_assert(chan_l); /* Assign an ID and bump the counter */ - chan_l->global_identifier = n_channels_allocated++; + chan_l->global_identifier = ++n_channels_allocated; /* Timestamp it */ channel_listener_timestamp_created(chan_l); @@ -898,6 +1002,11 @@ channel_free(channel_t *chan) circuitmux_set_policy(chan->cmux, NULL); } + /* Remove all timers and associated handle entries now */ + timer_free(chan->padding_timer); + channel_handle_free(chan->timer_handle); + channel_handles_clear(chan); + /* Call a free method if there is one */ if (chan->free_fn) chan->free_fn(chan); @@ -976,6 +1085,11 @@ channel_force_free(channel_t *chan) circuitmux_set_policy(chan->cmux, NULL); } + /* Remove all timers and associated handle entries now */ + timer_free(chan->padding_timer); + channel_handle_free(chan->timer_handle); + channel_handles_clear(chan); + /* Call a free method if there is one */ if (chan->free_fn) chan->free_fn(chan); @@ -2595,6 +2709,19 @@ channel_do_open_actions(channel_t *chan) } } + /* Disable or reduce padding according to user prefs. */ + if (chan->padding_enabled || get_options()->ConnectionPadding == 1) { + if (!get_options()->ConnectionPadding) { + channelpadding_disable_padding_on_channel(chan); + } + + /* Padding can be forced and/or reduced by clients, regardless of if + * the channel supports it */ + if (get_options()->ReducedConnectionPadding) { + channelpadding_reduce_padding_on_channel(chan); + } + } + circuit_n_chan_done(chan, 1, close_origin_circuits); } @@ -3232,6 +3359,11 @@ channel_free_all(void) /* Geez, anything still left over just won't die ... let it leak then */ HT_CLEAR(channel_idmap, &channel_identity_map); + /* Same with channel_gid_map */ + log_debug(LD_CHANNEL, + "Freeing channel_gid_map"); + HT_CLEAR(channel_gid_map, &channel_gid_map); + log_debug(LD_CHANNEL, "Done cleaning up after channels"); } @@ -3267,22 +3399,20 @@ channel_connect(const tor_addr_t *addr, uint16_t port, */ int -channel_is_better(time_t now, channel_t *a, channel_t *b, - int forgive_new_connections) +channel_is_better(channel_t *a, channel_t *b) { - int a_grace, b_grace; int a_is_canonical, b_is_canonical; - int a_has_circs, b_has_circs; - - /* - * Do not definitively deprecate a new channel with no circuits on it - * until this much time has passed. - */ -#define NEW_CHAN_GRACE_PERIOD (15*60) tor_assert(a); tor_assert(b); + /* If one channel is bad for new circuits, and the other isn't, + * use the one that is still good. */ + if (!channel_is_bad_for_new_circs(a) && channel_is_bad_for_new_circs(b)) + return 1; + if (channel_is_bad_for_new_circs(a) && !channel_is_bad_for_new_circs(b)) + return 0; + /* Check if one is canonical and the other isn't first */ a_is_canonical = channel_is_canonical(a); b_is_canonical = channel_is_canonical(b); @@ -3290,26 +3420,31 @@ channel_is_better(time_t now, channel_t *a, channel_t *b, if (a_is_canonical && !b_is_canonical) return 1; if (!a_is_canonical && b_is_canonical) return 0; + /* Check if we suspect that one of the channels will be preferred + * by the peer */ + if (a->is_canonical_to_peer && !b->is_canonical_to_peer) return 1; + if (!a->is_canonical_to_peer && b->is_canonical_to_peer) return 0; + /* - * Okay, if we're here they tied on canonicity. Next we check if - * they have any circuits, and if one does and the other doesn't, - * we prefer the one that does, unless we are forgiving and the - * one that has no circuits is in its grace period. + * Okay, if we're here they tied on canonicity, the prefer the older + * connection, so that the adversary can't create a new connection + * and try to switch us over to it (which will leak information + * about long-lived circuits). Additionally, switching connections + * too often makes us more vulnerable to attacks like Torscan and + * passive netflow-based equivalents. + * + * Connections will still only live for at most a week, due to + * the check in connection_or_group_set_badness() against + * TIME_BEFORE_OR_CONN_IS_TOO_OLD, which marks old connections as + * unusable for new circuits after 1 week. That check sets + * is_bad_for_new_circs, which is checked in channel_get_for_extend(). + * + * We check channel_is_bad_for_new_circs() above here anyway, for safety. */ + if (channel_when_created(a) < channel_when_created(b)) return 1; + else if (channel_when_created(a) > channel_when_created(b)) return 0; - a_has_circs = (channel_num_circuits(a) > 0); - b_has_circs = (channel_num_circuits(b) > 0); - a_grace = (forgive_new_connections && - (now < channel_when_created(a) + NEW_CHAN_GRACE_PERIOD)); - b_grace = (forgive_new_connections && - (now < channel_when_created(b) + NEW_CHAN_GRACE_PERIOD)); - - if (a_has_circs && !b_has_circs && !b_grace) return 1; - if (!a_has_circs && b_has_circs && !a_grace) return 0; - - /* They tied on circuits too; just prefer whichever is newer */ - - if (channel_when_created(a) > channel_when_created(b)) return 1; + if (channel_num_circuits(a) > channel_num_circuits(b)) return 1; else return 0; } @@ -3334,7 +3469,6 @@ channel_get_for_extend(const char *rsa_id_digest, channel_t *chan, *best = NULL; int n_inprogress_goodaddr = 0, n_old = 0; int n_noncanonical = 0, n_possible = 0; - time_t now = approx_time(); tor_assert(msg_out); tor_assert(launch_out); @@ -3404,7 +3538,7 @@ channel_get_for_extend(const char *rsa_id_digest, continue; } - if (channel_is_better(now, chan, best, 0)) + if (channel_is_better(chan, best)) best = chan; } @@ -4186,8 +4320,12 @@ channel_timestamp_active(channel_t *chan) time_t now = time(NULL); tor_assert(chan); + chan->timestamp_xfer_ms = monotime_coarse_absolute_msec(); chan->timestamp_active = now; + + /* Clear any potential netflow padding timer. We're active */ + chan->next_padding_time_ms = 0; } /** @@ -4270,11 +4408,14 @@ void channel_timestamp_recv(channel_t *chan) { time_t now = time(NULL); - tor_assert(chan); + chan->timestamp_xfer_ms = monotime_coarse_absolute_msec(); chan->timestamp_active = now; chan->timestamp_recv = now; + + /* Clear any potential netflow padding timer. We're active */ + chan->next_padding_time_ms = 0; } /** @@ -4287,11 +4428,15 @@ void channel_timestamp_xmit(channel_t *chan) { time_t now = time(NULL); - tor_assert(chan); + chan->timestamp_xfer_ms = monotime_coarse_absolute_msec(); + chan->timestamp_active = now; chan->timestamp_xmit = now; + + /* Clear any potential netflow padding timer. We're active */ + chan->next_padding_time_ms = 0; } /*************************************************************** diff --git a/src/or/channel.h b/src/or/channel.h index 33dceb1be0..ea280f2fd2 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -11,6 +11,8 @@ #include "or.h" #include "circuitmux.h" +#include "timers.h" +#include "handles.h" /* Channel handler function pointer typedefs */ typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *); @@ -22,6 +24,17 @@ TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s); typedef struct chan_cell_queue chan_cell_queue_t; /** + * This enum is used by channelpadding to decide when to pad channels. + * Don't add values to it without updating the checks in + * channelpadding_decide_to_pad_channel(). + */ +typedef enum { + CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS = 0, + CHANNEL_USED_FOR_FULL_CIRCS, + CHANNEL_USED_FOR_USER_TRAFFIC, +} channel_usage_info_t; + +/** * Channel struct; see the channel_t typedef in or.h. A channel is an * abstract interface for the OR-to-OR connection, similar to connection_or_t, * but without the strong coupling to the underlying TLS implementation. They @@ -34,11 +47,17 @@ struct channel_s { /** Magic number for type-checking cast macros */ uint32_t magic; + /** List entry for hashtable for global-identifier lookup. */ + HT_ENTRY(channel_s) gidmap_node; + + /** Handle entry for handle-based lookup */ + HANDLE_ENTRY(channel, channel_s); + /** Current channel state */ channel_state_t state; /** Globally unique ID number for a channel over the lifetime of a Tor - * process. + * process. This may not be 0. */ uint64_t global_identifier; @@ -48,6 +67,61 @@ struct channel_s { /** has this channel ever been open? */ unsigned int has_been_open:1; + /** + * This field indicates if the other side has enabled or disabled + * padding via either the link protocol version or + * channelpadding_negotiate cells. + * + * Clients can override this with ConnectionPadding in torrc to + * disable or force padding to relays, but relays cannot override the + * client's request. + */ + unsigned int padding_enabled:1; + + /** Cached value of our decision to pad (to avoid expensive + * checks during critical path statistics counting). */ + unsigned int currently_padding:1; + + /** Is there a pending netflow padding callback? */ + unsigned int pending_padding_callback:1; + + /** Is our peer likely to consider this channel canonical? */ + unsigned int is_canonical_to_peer:1; + + /** Has this channel ever been used for non-directory traffic? + * Used to decide what channels to pad, and when. */ + channel_usage_info_t channel_usage; + + /** When should we send a cell for netflow padding, in absolute + * milliseconds since monotime system start. 0 means no padding + * is scheduled. */ + uint64_t next_padding_time_ms; + + /** The callback pointer for the padding callbacks */ + tor_timer_t *padding_timer; + /** The handle to this channel (to free on canceled timers) */ + struct channel_handle_t *timer_handle; + + /** + * These two fields specify the minimum and maximum negotiated timeout + * values for inactivity (send or receive) before we decide to pad a + * channel. These fields can be set either via a PADDING_NEGOTIATE cell, + * or the torrc option ReducedConnectionPadding. The consensus parameters + * nf_ito_low and nf_ito_high are used to ensure that padding can only be + * negotiated to be less frequent than what is specified in the consensus. + * (This is done to prevent wingnut clients from requesting excessive + * padding). + * + * The actual timeout value is randomly chosen between these two values + * as per the table in channelpadding_get_netflow_inactive_timeout_ms(), + * after ensuring that these values do not specify lower timeouts than + * the consensus parameters. + * + * If these are 0, we have not negotiated or specified custom padding + * times, and instead use consensus defaults. */ + uint16_t padding_timeout_low_ms; + uint16_t padding_timeout_high_ms; + /** Why did we close? */ enum { @@ -87,6 +161,18 @@ struct channel_s { time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ + /** + * This is a high-resolution monotonic timestamp that marks when we + * believe the channel has actually sent or received data to/from + * the wire. Right now, it is used to determine when we should send + * a padding cell for channelpadding. + * + * XXX: Are we setting timestamp_xfer_ms in the right places to + * accurately reflect actual network data transfer? Or might this be + * very wrong wrt when bytes actually go on the wire? + */ + uint64_t timestamp_xfer_ms; + /* Methods implemented by the lower layer */ /** Free a channel */ @@ -214,8 +300,8 @@ struct channel_s { unsigned int is_bad_for_new_circs:1; /** True iff we have decided that the other end of this connection - * is a client. Channels with this flag set should never be used - * to satisfy an EXTEND request. */ + * is a client or bridge relay. Connections with this flag set should never + * be used to satisfy an EXTEND request. */ unsigned int is_client:1; /** Set if the channel was initiated remotely (came from a listener) */ @@ -516,9 +602,7 @@ channel_t * channel_get_for_extend(const char *rsa_id_digest, int *launch_out); /* Ask which of two channels is better for circuit-extension purposes */ -int channel_is_better(time_t now, - channel_t *a, channel_t *b, - int forgive_new_connections); +int channel_is_better(channel_t *a, channel_t *b); /** Channel lookups */ @@ -601,6 +685,7 @@ void channel_listener_dump_statistics(channel_listener_t *chan_l, int severity); void channel_listener_dump_transport_statistics(channel_listener_t *chan_l, int severity); +void channel_check_for_duplicates(void); void channel_update_bad_for_new_circs(const char *digest, int force); @@ -630,5 +715,8 @@ int packed_cell_is_destroy(channel_t *chan, const packed_cell_t *packed_cell, circid_t *circid_out); +/* Declare the handle helpers */ +HANDLE_DECL(channel, channel_s,) + #endif diff --git a/src/or/channelpadding.c b/src/or/channelpadding.c new file mode 100644 index 0000000000..f5248e8960 --- /dev/null +++ b/src/or/channelpadding.c @@ -0,0 +1,749 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2015, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of + * channelpadding_channel_to_channelinfo() */ +#define TOR_CHANNEL_INTERNAL_ + +#include "or.h" +#include "channel.h" +#include "channelpadding.h" +#include "channeltls.h" +#include "config.h" +#include "networkstatus.h" +#include "connection.h" +#include "connection_or.h" +#include "main.h" +#include "rephist.h" +#include "router.h" +#include "compat_time.h" +#include <event2/event.h> + +STATIC int channelpadding_get_netflow_inactive_timeout_ms(const channel_t *); +STATIC int channelpadding_send_disable_command(channel_t *); +STATIC int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *); + +/** The total number of pending channelpadding timers */ +static uint64_t total_timers_pending; + +/** These are cached consensus parameters for netflow */ +/** The timeout lower bound that is allowed before sending padding */ +static int consensus_nf_ito_low; +/** The timeout upper bound that is allowed before sending padding */ +static int consensus_nf_ito_high; +/** The timeout lower bound that is allowed before sending reduced padding */ +static int consensus_nf_ito_low_reduced; +/** The timeout upper bound that is allowed before sending reduced padding */ +static int consensus_nf_ito_high_reduced; +/** The connection timeout between relays */ +static int consensus_nf_conntimeout_relays; +/** The connection timeout for client connections */ +static int consensus_nf_conntimeout_clients; +/** Should we pad before circuits are actually used for client data? */ +static int consensus_nf_pad_before_usage; +/** Should we pad relay-to-relay connections? */ +static int consensus_nf_pad_relays; + +#define TOR_MSEC_PER_SEC 1000 +#define TOR_USEC_PER_MSEC 1000 + +/** + * How often do we get called by the connection housekeeping (ie: once + * per second) */ +#define TOR_HOUSEKEEPING_CALLBACK_MSEC 1000 +/** + * Additional extra time buffer on the housekeeping callback, since + * it can be delayed. This extra slack is used to decide if we should + * schedule a timer or wait for the next callback. */ +#define TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC 100 + +/** + * This macro tells us if either end of the channel is connected to a client. + * (If we're not a server, we're definitely a client. If the channel thinks + * its a client, use that. Then finally verify in the consensus). + */ +#define CHANNEL_IS_CLIENT(chan, options) \ + (!public_server_mode((options)) || (chan)->is_client || \ + !connection_or_digest_is_known_relay((chan)->identity_digest)) + +/** + * This function is called to update cached consensus parameters every time + * there is a consensus update. This allows us to move the consensus param + * search off of the critical path, so it does not need to be evaluated + * for every single connection, every second. + */ +void +channelpadding_new_consensus_params(networkstatus_t *ns) +{ +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW 1500 +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH 9500 +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN 0 +#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX 60000 + consensus_nf_ito_low = networkstatus_get_param(ns, "nf_ito_low", + DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW, + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN, + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX); + consensus_nf_ito_high = networkstatus_get_param(ns, "nf_ito_high", + DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH, + consensus_nf_ito_low, + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX); + +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW 9000 +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH 14000 +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN 0 +#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX 60000 + consensus_nf_ito_low_reduced = + networkstatus_get_param(ns, "nf_ito_low_reduced", + DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW, + DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN, + DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX); + + consensus_nf_ito_high_reduced = + networkstatus_get_param(ns, "nf_ito_high_reduced", + DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH, + consensus_nf_ito_low_reduced, + DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX); + +#define CONNTIMEOUT_RELAYS_DFLT (60*60) // 1 hour +#define CONNTIMEOUT_RELAYS_MIN 60 +#define CONNTIMEOUT_RELAYS_MAX (7*24*60*60) // 1 week + consensus_nf_conntimeout_relays = + networkstatus_get_param(ns, "nf_conntimeout_relays", + CONNTIMEOUT_RELAYS_DFLT, + CONNTIMEOUT_RELAYS_MIN, + CONNTIMEOUT_RELAYS_MAX); + +#define CIRCTIMEOUT_CLIENTS_DFLT (30*60) // 30 minutes +#define CIRCTIMEOUT_CLIENTS_MIN 60 +#define CIRCTIMEOUT_CLIENTS_MAX (24*60*60) // 24 hours + consensus_nf_conntimeout_clients = + networkstatus_get_param(ns, "nf_conntimeout_clients", + CIRCTIMEOUT_CLIENTS_DFLT, + CIRCTIMEOUT_CLIENTS_MIN, + CIRCTIMEOUT_CLIENTS_MAX); + + consensus_nf_pad_before_usage = + networkstatus_get_param(ns, "nf_pad_before_usage", 1, 0, 1); + + consensus_nf_pad_relays = + networkstatus_get_param(ns, "nf_pad_relays", 0, 0, 1); +} + +/** + * Get a random netflow inactive timeout keepalive period in milliseconds, + * the range for which is determined by consensus parameters, negotiation, + * configuration, or default values. The consensus parameters enforce the + * minimum possible value, to avoid excessively frequent padding. + * + * The ranges for this value were chosen to be low enough to ensure that + * routers do not emit a new netflow record for a connection due to it + * being idle. + * + * Specific timeout values for major routers are listed in Proposal 251. + * No major router appeared capable of setting an inactive timeout below 10 + * seconds, so we set the defaults below that value, since we can always + * scale back if it ends up being too much padding. + * + * Returns the next timeout period (in milliseconds) after which we should + * send a padding packet, or 0 if padding is disabled. + */ +STATIC int +channelpadding_get_netflow_inactive_timeout_ms(const channel_t *chan) +{ + int low_timeout = consensus_nf_ito_low; + int high_timeout = consensus_nf_ito_high; + int X1, X2; + + if (low_timeout == 0 && low_timeout == high_timeout) + return 0; // No padding + + /* If we have negotiated different timeout values, use those, but + * don't allow them to be lower than the consensus ones */ + if (chan->padding_timeout_low_ms && chan->padding_timeout_high_ms) { + low_timeout = MAX(low_timeout, chan->padding_timeout_low_ms); + high_timeout = MAX(high_timeout, chan->padding_timeout_high_ms); + } + + if (low_timeout == high_timeout) + return low_timeout; // No randomization + + /* + * This MAX() hack is here because we apply the timeout on both the client + * and the server. This creates the situation where the total time before + * sending a packet in either direction is actually + * min(client_timeout,server_timeout). + * + * If X is a random variable uniform from 0..R-1 (where R=high-low), + * then Y=max(X,X) has Prob(Y == i) = (2.0*i + 1)/(R*R). + * + * If we create a third random variable Z=min(Y,Y), then it turns out that + * Exp[Z] ~= Exp[X]. Here's a table: + * + * R Exp[X] Exp[Z] Exp[min(X,X)] Exp[max(X,X)] + * 2000 999.5 1066 666.2 1332.8 + * 3000 1499.5 1599.5 999.5 1999.5 + * 5000 2499.5 2666 1666.2 3332.8 + * 6000 2999.5 3199.5 1999.5 3999.5 + * 7000 3499.5 3732.8 2332.8 4666.2 + * 8000 3999.5 4266.2 2666.2 5332.8 + * 10000 4999.5 5328 3332.8 6666.2 + * 15000 7499.5 7995 4999.5 9999.5 + * 20000 9900.5 10661 6666.2 13332.8 + * + * In other words, this hack makes it so that when both the client and + * the guard are sending this padding, then the averages work out closer + * to the midpoint of the range, making the overhead easier to tune. + * If only one endpoint is padding (for example: if the relay does not + * support padding, but the client has set ConnectionPadding 1; or + * if the relay does support padding, but the client has set + * ReducedConnectionPadding 1), then the defense will still prevent + * record splitting, but with less overhead than the midpoint + * (as seen by the Exp[max(X,X)] column). + * + * To calculate average padding packet frequency (and thus overhead), + * index into the table by picking a row based on R = high-low. Then, + * use the appropriate column (Exp[Z] for two-sided padding, and + * Exp[max(X,X)] for one-sided padding). Finally, take this value + * and add it to the low timeout value. This value is the average + * frequency which padding packets will be sent. + */ + + X1 = crypto_rand_int(high_timeout - low_timeout); + X2 = crypto_rand_int(high_timeout - low_timeout); + return low_timeout + MAX(X1, X2); +} + +/** + * Update this channel's padding settings based on the PADDING_NEGOTIATE + * contents. + * + * Returns -1 on error; 1 on success. + */ +int +channelpadding_update_padding_for_channel(channel_t *chan, + const channelpadding_negotiate_t *pad_vars) +{ + if (pad_vars->version != 0) { + static ratelim_t version_limit = RATELIM_INIT(600); + + log_fn_ratelim(&version_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL, + "Got a PADDING_NEGOTIATE cell with an unknown version. Ignoring."); + return -1; + } + + // We should not allow malicious relays to disable or reduce padding for + // us as clients. In fact, we should only accept this cell at all if we're + // operating as a relay. Bridges should not accept it from relays, either + // (only from their clients). + if ((get_options()->BridgeRelay && + connection_or_digest_is_known_relay(chan->identity_digest)) || + !get_options()->ORPort_set) { + static ratelim_t relay_limit = RATELIM_INIT(600); + + log_fn_ratelim(&relay_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL, + "Got a PADDING_NEGOTIATE from relay at %s (%s). " + "This should not happen.", + chan->get_remote_descr(chan, 0), + hex_str(chan->identity_digest, DIGEST_LEN)); + return -1; + } + + chan->padding_enabled = (pad_vars->command == CHANNELPADDING_COMMAND_START); + + /* Min must not be lower than the current consensus parameter + nf_ito_low. */ + chan->padding_timeout_low_ms = MAX(consensus_nf_ito_low, + pad_vars->ito_low_ms); + + /* Max must not be lower than ito_low_ms */ + chan->padding_timeout_high_ms = MAX(chan->padding_timeout_low_ms, + pad_vars->ito_high_ms); + + log_fn(LOG_INFO,LD_OR, + "Negotiated padding=%d, lo=%d, hi=%d on "U64_FORMAT, + chan->padding_enabled, chan->padding_timeout_low_ms, + chan->padding_timeout_high_ms, + U64_PRINTF_ARG(chan->global_identifier)); + + return 1; +} + +/** + * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side not + * to send padding. + * + * Returns -1 on error, 0 on success. + */ +STATIC int +channelpadding_send_disable_command(channel_t *chan) +{ + channelpadding_negotiate_t disable; + cell_t cell; + + tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >= + MIN_LINK_PROTO_FOR_CHANNEL_PADDING); + + memset(&cell, 0, sizeof(cell_t)); + memset(&disable, 0, sizeof(channelpadding_negotiate_t)); + cell.command = CELL_PADDING_NEGOTIATE; + + channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP); + + if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, + &disable) < 0) + return -1; + + if (chan->write_cell(chan, &cell) == 1) + return 0; + else + return -1; +} + +/** + * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side to + * resume sending padding at some rate. + * + * Returns -1 on error, 0 on success. + */ +int +channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout, + uint16_t high_timeout) +{ + channelpadding_negotiate_t enable; + cell_t cell; + + tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >= + MIN_LINK_PROTO_FOR_CHANNEL_PADDING); + + memset(&cell, 0, sizeof(cell_t)); + memset(&enable, 0, sizeof(channelpadding_negotiate_t)); + cell.command = CELL_PADDING_NEGOTIATE; + + channelpadding_negotiate_set_command(&enable, CHANNELPADDING_COMMAND_START); + channelpadding_negotiate_set_ito_low_ms(&enable, low_timeout); + channelpadding_negotiate_set_ito_high_ms(&enable, high_timeout); + + if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, + &enable) < 0) + return -1; + + if (chan->write_cell(chan, &cell) == 1) + return 0; + else + return -1; +} + +/** + * Sends a CELL_PADDING cell on a channel if it has been idle since + * our callback was scheduled. + * + * This function also clears the pending padding timer and the callback + * flags. + */ +static void +channelpadding_send_padding_cell_for_callback(channel_t *chan) +{ + cell_t cell; + + /* Check that the channel is still valid and open */ + if (!chan || chan->state != CHANNEL_STATE_OPEN) { + if (chan) chan->pending_padding_callback = 0; + log_fn(LOG_INFO,LD_OR, + "Scheduled a netflow padding cell, but connection already closed."); + return; + } + + /* We should have a pending callback flag set. */ + if (BUG(chan->pending_padding_callback == 0)) + return; + + chan->pending_padding_callback = 0; + + if (!chan->next_padding_time_ms || + chan->has_queued_writes(chan)) { + /* We must have been active before the timer fired */ + chan->next_padding_time_ms = 0; + return; + } + + { + uint64_t now = monotime_coarse_absolute_msec(); + + log_fn(LOG_INFO,LD_OR, + "Sending netflow keepalive on "U64_FORMAT" to %s (%s) after " + I64_FORMAT" ms. Delta "I64_FORMAT"ms", + U64_PRINTF_ARG(chan->global_identifier), + safe_str_client(chan->get_remote_descr(chan, 0)), + safe_str_client(hex_str(chan->identity_digest, DIGEST_LEN)), + U64_PRINTF_ARG(now - chan->timestamp_xfer_ms), + U64_PRINTF_ARG(now - chan->next_padding_time_ms)); + } + + /* Clear the timer */ + chan->next_padding_time_ms = 0; + + /* Send the padding cell. This will cause the channel to get a + * fresh timestamp_active */ + memset(&cell, 0, sizeof(cell)); + cell.command = CELL_PADDING; + chan->write_cell(chan, &cell); +} + +/** + * tor_timer callback function for us to send padding on an idle channel. + * + * This function just obtains the channel from the callback handle, ensures + * it is still valid, and then hands it off to + * channelpadding_send_padding_cell_for_callback(), which checks if + * the channel is still idle before sending padding. + */ +static void +channelpadding_send_padding_callback(tor_timer_t *timer, void *args, + const struct monotime_t *when) +{ + channel_t *chan = channel_handle_get((struct channel_handle_t*)args); + (void)timer; (void)when; + + if (chan && CHANNEL_CAN_HANDLE_CELLS(chan)) { + /* Hrmm.. It might be nice to have an equivalent to assert_connection_ok + * for channels. Then we could get rid of the channeltls dependency */ + tor_assert(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn)->magic == + OR_CONNECTION_MAGIC); + assert_connection_ok(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), approx_time()); + + channelpadding_send_padding_cell_for_callback(chan); + } else { + log_fn(LOG_INFO,LD_OR, + "Channel closed while waiting for timer."); + } + + total_timers_pending--; +} + +/** + * Schedules a callback to send padding on a channel in_ms milliseconds from + * now. + * + * Returns CHANNELPADDING_WONTPAD on error, CHANNELPADDING_PADDING_SENT if we + * sent the packet immediately without a timer, and + * CHANNELPADDING_PADDING_SCHEDULED if we decided to schedule a timer. + */ +static channelpadding_decision_t +channelpadding_schedule_padding(channel_t *chan, int in_ms) +{ + struct timeval timeout; + tor_assert(!chan->pending_padding_callback); + + if (in_ms <= 0) { + chan->pending_padding_callback = 1; + channelpadding_send_padding_cell_for_callback(chan); + return CHANNELPADDING_PADDING_SENT; + } + + timeout.tv_sec = in_ms/TOR_MSEC_PER_SEC; + timeout.tv_usec = (in_ms%TOR_USEC_PER_MSEC)*TOR_USEC_PER_MSEC; + + if (!chan->timer_handle) { + chan->timer_handle = channel_handle_new(chan); + } + + if (chan->padding_timer) { + timer_set_cb(chan->padding_timer, + channelpadding_send_padding_callback, + chan->timer_handle); + } else { + chan->padding_timer = timer_new(channelpadding_send_padding_callback, + chan->timer_handle); + } + timer_schedule(chan->padding_timer, &timeout); + + rep_hist_padding_count_timers(++total_timers_pending); + + chan->pending_padding_callback = 1; + return CHANNELPADDING_PADDING_SCHEDULED; +} + +/** + * Calculates the number of milliseconds from now to schedule a padding cell. + * + * Returns the number of milliseconds from now (relative) to schedule the + * padding callback. If the padding timer is more than 1.1 seconds in the + * future, we return -1, to avoid scheduling excessive callbacks. If padding + * is disabled in the consensus, we return -2. + * + * Side-effects: Updates chan->next_padding_time_ms, storing an (absolute, not + * relative) millisecond representation of when we should send padding, unless + * other activity happens first. This side-effect allows us to avoid + * scheduling a libevent callback until we're within 1.1 seconds of the padding + * time. + */ +#define CHANNELPADDING_TIME_LATER -1 +#define CHANNELPADDING_TIME_DISABLED -2 +STATIC int64_t +channelpadding_compute_time_until_pad_for_netflow(channel_t *chan) +{ + uint64_t long_now = monotime_coarse_absolute_msec(); + + if (!chan->next_padding_time_ms) { + /* If the below line or crypto_rand_int() shows up on a profile, + * we can avoid getting a timeout until we're at least nf_ito_lo + * from a timeout window. That will prevent us from setting timers + * on connections that were active up to 1.5 seconds ago. + * Idle connections should only call this once every 5.5s on average + * though, so that might be a micro-optimization for little gain. */ + int64_t padding_timeout = + channelpadding_get_netflow_inactive_timeout_ms(chan); + + if (!padding_timeout) + return CHANNELPADDING_TIME_DISABLED; + + chan->next_padding_time_ms = padding_timeout + + chan->timestamp_xfer_ms; + } + + /* If the next padding time is beyond the maximum possible consensus value, + * then this indicates a clock jump, so just send padding now. This is + * better than using monotonic time because we want to avoid the situation + * where we wait around forever for monotonic time to move forward after + * a clock jump far into the past. + */ + if (chan->next_padding_time_ms > long_now + + DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX) { + tor_fragile_assert(); + log_warn(LD_BUG, + "Channel padding timeout scheduled "I64_FORMAT"ms in the future. " + "Did the monotonic clock just jump?", + I64_PRINTF_ARG(chan->next_padding_time_ms - long_now)); + return 0; /* Clock jumped: Send padding now */ + } + + /* If the timeout will expire before the next time we're called (1000ms + from now, plus some slack), then calculate the number of milliseconds + from now which we should send padding, so we can schedule a callback + then. + */ + if (long_now + + (TOR_HOUSEKEEPING_CALLBACK_MSEC + TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC) + >= chan->next_padding_time_ms) { + int64_t ms_until_pad_for_netflow = chan->next_padding_time_ms - + long_now; + if (ms_until_pad_for_netflow < 0) { + log_warn(LD_BUG, + "Channel padding timeout scheduled "I64_FORMAT"ms in the past. " + "Did the monotonic clock just jump?", + I64_PRINTF_ARG(-ms_until_pad_for_netflow)); + return 0; /* Clock jumped: Send padding now */ + } + + return ms_until_pad_for_netflow; + } + return CHANNELPADDING_TIME_LATER; +} + +/** + * Returns a randomized value for channel idle timeout in seconds. + * The channel idle timeout governs how quickly we close a channel + * after its last circuit has disappeared. + * + * There are three classes of channels: + * 1. Client+non-canonical. These live for 3-4.5 minutes + * 2. relay to relay. These live for 45-75 min by default + * 3. Reduced padding clients. These live for 1.5-2.25 minutes. + * + * Also allows the default relay-to-relay value to be controlled by the + * consensus. + */ +unsigned int +channelpadding_get_channel_idle_timeout(const channel_t *chan, + int is_canonical) +{ + const or_options_t *options = get_options(); + unsigned int timeout; + + /* Non-canonical and client channels only last for 3-4.5 min when idle */ + if (!is_canonical || CHANNEL_IS_CLIENT(chan, options)) { +#define CONNTIMEOUT_CLIENTS_BASE 180 // 3 to 4.5 min + timeout = CONNTIMEOUT_CLIENTS_BASE + + crypto_rand_int(CONNTIMEOUT_CLIENTS_BASE/2); + } else { // Canonical relay-to-relay channels + // 45..75min or consensus +/- 25% + timeout = consensus_nf_conntimeout_relays; + timeout = 3*timeout/4 + crypto_rand_int(timeout/2); + } + + /* If ReducedConnectionPadding is set, we want to halve the duration of + * the channel idle timeout, since reducing the additional time that + * a channel stays open will reduce the total overhead for making + * new channels. This reduction in overhead/channel expense + * is important for mobile users. The option cannot be set by relays. + * + * We also don't reduce any values for timeout that the user explicitly + * set. + */ + if (options->ReducedConnectionPadding + && !options->CircuitsAvailableTimeout) { + timeout /= 2; + } + + return timeout; +} + +/** + * This function controls how long we keep idle circuits open, + * and how long we build predicted circuits. This behavior is under + * the control of channelpadding because circuit availability is the + * dominant factor in channel lifespan, which influences total padding + * overhead. + * + * Returns a randomized number of seconds in a range from + * CircuitsAvailableTimeout to 2*CircuitsAvailableTimeout. This value is halved + * if ReducedConnectionPadding is set. The default value of + * CircuitsAvailableTimeout can be controlled by the consensus. + */ +int +channelpadding_get_circuits_available_timeout(void) +{ + const or_options_t *options = get_options(); + int timeout = options->CircuitsAvailableTimeout; + + if (!timeout) { + timeout = consensus_nf_conntimeout_clients; + + /* If ReducedConnectionPadding is set, we want to halve the duration of + * the channel idle timeout, since reducing the additional time that + * a channel stays open will reduce the total overhead for making + * new connections. This reduction in overhead/connection expense + * is important for mobile users. The option cannot be set by relays. + * + * We also don't reduce any values for timeout that the user explicitly + * set. + */ + if (options->ReducedConnectionPadding) { + // half the value to 15..30min by default + timeout /= 2; + } + } + + // 30..60min by default + timeout = timeout + crypto_rand_int(timeout); + + return timeout; +} + +/** + * Calling this function on a channel causes it to tell the other side + * not to send padding, and disables sending padding from this side as well. + */ +void +channelpadding_disable_padding_on_channel(channel_t *chan) +{ + chan->padding_enabled = 0; + + // Send cell to disable padding on the other end + channelpadding_send_disable_command(chan); +} + +/** + * Calling this function on a channel causes it to tell the other side + * not to send padding, and reduces the rate that padding is sent from + * this side. + */ +void +channelpadding_reduce_padding_on_channel(channel_t *chan) +{ + /* Padding can be forced and reduced by clients, regardless of if + * the channel supports it. So we check for support here before + * sending any commands. */ + if (chan->padding_enabled) { + channelpadding_send_disable_command(chan); + } + + chan->padding_timeout_low_ms = consensus_nf_ito_low_reduced; + chan->padding_timeout_high_ms = consensus_nf_ito_high_reduced; + + log_fn(LOG_INFO,LD_OR, + "Reduced padding on channel "U64_FORMAT": lo=%d, hi=%d", + U64_PRINTF_ARG(chan->global_identifier), + chan->padding_timeout_low_ms, chan->padding_timeout_high_ms); +} + +/** + * This function is called once per second by run_connection_housekeeping(), + * but only if the channel is still open, valid, and non-wedged. + * + * It decides if and when we should send a padding cell, and if needed, + * schedules a callback to send that cell at the appropriate time. + * + * Returns an enum that represents the current padding decision state. + * Return value is currently used only by unit tests. + */ +channelpadding_decision_t +channelpadding_decide_to_pad_channel(channel_t *chan) +{ + const or_options_t *options = get_options(); + + /* Only pad open channels */ + if (chan->state != CHANNEL_STATE_OPEN) + return CHANNELPADDING_WONTPAD; + + if (chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS) { + if (!consensus_nf_pad_before_usage) + return CHANNELPADDING_WONTPAD; + } else if (chan->channel_usage != CHANNEL_USED_FOR_USER_TRAFFIC) { + return CHANNELPADDING_WONTPAD; + } + + if (chan->pending_padding_callback) + return CHANNELPADDING_PADDING_ALREADY_SCHEDULED; + + /* Don't pad the channel if we didn't negotiate it, but still + * allow clients to force padding if options->ChannelPadding is + * explicitly set to 1. + */ + if (!chan->padding_enabled && options->ConnectionPadding != 1) { + return CHANNELPADDING_WONTPAD; + } + + if (!chan->has_queued_writes(chan)) { + int is_client_channel = 0; + + if (CHANNEL_IS_CLIENT(chan, options)) { + is_client_channel = 1; + } + + /* If nf_pad_relays=1 is set in the consensus, we pad + * on *all* idle connections, relay-relay or relay-client. + * Otherwise pad only for client+bridge cons */ + if (is_client_channel || consensus_nf_pad_relays) { + int64_t pad_time_ms = + channelpadding_compute_time_until_pad_for_netflow(chan); + + if (pad_time_ms == CHANNELPADDING_TIME_DISABLED) { + return CHANNELPADDING_WONTPAD; + } else if (pad_time_ms == CHANNELPADDING_TIME_LATER) { + chan->currently_padding = 1; + return CHANNELPADDING_PADLATER; + } else { + if (BUG(pad_time_ms > INT_MAX)) { + pad_time_ms = INT_MAX; + } + /* We have to schedule a callback because we're called exactly once per + * second, but we don't want padding packets to go out exactly on an + * integer multiple of seconds. This callback will only be scheduled + * if we're within 1.1 seconds of the padding time. + */ + chan->currently_padding = 1; + return channelpadding_schedule_padding(chan, (int)pad_time_ms); + } + } else { + chan->currently_padding = 0; + return CHANNELPADDING_WONTPAD; + } + } else { + return CHANNELPADDING_PADLATER; + } +} + diff --git a/src/or/channelpadding.h b/src/or/channelpadding.h new file mode 100644 index 0000000000..2708ee9739 --- /dev/null +++ b/src/or/channelpadding.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2015, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ +#ifndef TOR_CHANNELPADDING_H +#define TOR_CHANNELPADDING_H + +#include "channelpadding_negotiation.h" + +typedef enum { + CHANNELPADDING_WONTPAD, + CHANNELPADDING_PADLATER, + CHANNELPADDING_PADDING_SCHEDULED, + CHANNELPADDING_PADDING_ALREADY_SCHEDULED, + CHANNELPADDING_PADDING_SENT, +} channelpadding_decision_t; + +channelpadding_decision_t channelpadding_decide_to_pad_channel(channel_t + *chan); +int channelpadding_update_padding_for_channel(channel_t *, + const channelpadding_negotiate_t + *chan); + +void channelpadding_disable_padding_on_channel(channel_t *chan); +void channelpadding_reduce_padding_on_channel(channel_t *chan); +int channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout, + uint16_t high_timeout); + +int channelpadding_get_circuits_available_timeout(void); +unsigned int channelpadding_get_channel_idle_timeout(const channel_t *, int); +void channelpadding_new_consensus_params(networkstatus_t *ns); + +#endif + diff --git a/src/or/channeltls.c b/src/or/channeltls.c index 9d9e7446ab..f44e4fc8ea 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -57,6 +57,9 @@ #include "routerlist.h" #include "scheduler.h" #include "torcert.h" +#include "networkstatus.h" +#include "channelpadding_negotiation.h" +#include "channelpadding.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -122,6 +125,8 @@ static void channel_tls_process_netinfo_cell(cell_t *cell, static int command_allowed_before_handshake(uint8_t command); static int enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *tlschan); +static void channel_tls_process_padding_negotiate_cell(cell_t *cell, + channel_tls_t *chan); /** * Do parts of channel_tls_t initialization common to channel_tls_connect() @@ -734,6 +739,15 @@ channel_tls_matches_target_method(channel_t *chan, return 0; } + /* real_addr is the address this connection came from. + * base_.addr is updated by connection_or_init_conn_from_address() + * to be the address in the descriptor. It may be tempting to + * allow either address to be allowed, but if we did so, it would + * enable someone who steals a relay's keys to impersonate/MITM it + * from anywhere on the Internet! (Because they could make long-lived + * TLS connections from anywhere to all relays, and wait for them to + * be used for extends). + */ return tor_addr_eq(&(tlschan->conn->real_addr), target); } @@ -1098,9 +1112,16 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) /* We note that we're on the internet whenever we read a cell. This is * a fast operation. */ entry_guards_note_internet_connectivity(get_guard_selection_info()); + rep_hist_padding_count_read(PADDING_TYPE_TOTAL); + + if (TLS_CHAN_TO_BASE(chan)->currently_padding) + rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL); switch (cell->command) { case CELL_PADDING: + rep_hist_padding_count_read(PADDING_TYPE_CELL); + if (TLS_CHAN_TO_BASE(chan)->currently_padding) + rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL); ++stats_n_padding_cells_processed; /* do nothing */ break; @@ -1111,6 +1132,10 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) ++stats_n_netinfo_cells_processed; PROCESS_CELL(netinfo, cell, chan); break; + case CELL_PADDING_NEGOTIATE: + ++stats_n_netinfo_cells_processed; + PROCESS_CELL(padding_negotiate, cell, chan); + break; case CELL_CREATE: case CELL_CREATE_FAST: case CELL_CREATED: @@ -1566,9 +1591,12 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) /* We set this after sending the verions cell. */ /*XXXXX symbolic const.*/ - chan->base_.wide_circ_ids = + TLS_CHAN_TO_BASE(chan)->wide_circ_ids = chan->conn->link_proto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS; - chan->conn->wide_circ_ids = chan->base_.wide_circ_ids; + chan->conn->wide_circ_ids = TLS_CHAN_TO_BASE(chan)->wide_circ_ids; + + TLS_CHAN_TO_BASE(chan)->padding_enabled = + chan->conn->link_proto >= MIN_LINK_PROTO_FOR_CHANNEL_PADDING; if (send_certs) { if (connection_or_send_certs_cell(chan->conn) < 0) { @@ -1595,6 +1623,43 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) } /** + * Process a 'padding_negotiate' cell + * + * This function is called to handle an incoming PADDING_NEGOTIATE cell; + * enable or disable padding accordingly, and read and act on its timeout + * value contents. + */ +static void +channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) +{ + channelpadding_negotiate_t *negotiation; + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + + if (chan->conn->link_proto < MIN_LINK_PROTO_FOR_CHANNEL_PADDING) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a PADDING_NEGOTIATE cell on v%d connection; dropping.", + chan->conn->link_proto); + return; + } + + if (channelpadding_negotiate_parse(&negotiation, cell->payload, + CELL_PAYLOAD_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received malformed PADDING_NEGOTIATE cell on v%d connection; " + "dropping.", chan->conn->link_proto); + + return; + } + + channelpadding_update_padding_for_channel(TLS_CHAN_TO_BASE(chan), + negotiation); + + channelpadding_negotiate_free(negotiation); +} + +/** * Process a 'netinfo' cell * * This function is called to handle an incoming NETINFO cell; read and act @@ -1611,6 +1676,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) const uint8_t *cp, *end; uint8_t n_other_addrs; time_t now = time(NULL); + const routerinfo_t *me = router_get_my_routerinfo(); long apparent_skew = 0; tor_addr_t my_apparent_addr = TOR_ADDR_NULL; @@ -1654,6 +1720,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) tor_assert(tor_mem_is_zero( (const char*)(chan->conn->handshake_state-> authenticated_ed25519_peer_id.pubkey), 32)); + /* If the client never authenticated, it's a tor client or bridge + * relay, and we must not use it for EXTEND requests (nor could we, as + * there are no authenticated peer IDs) */ + channel_mark_client(TLS_CHAN_TO_BASE(chan)); channel_set_circid_type(TLS_CHAN_TO_BASE(chan), NULL, chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS); @@ -1689,8 +1759,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + + if (!get_options()->BridgeRelay && me && + get_uint32(my_addr_ptr) == htonl(me->addr)) { + TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; + } + } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); + + if (!get_options()->BridgeRelay && me && + !tor_addr_is_null(&me->ipv6_addr) && + tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { + TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; + } } n_other_addrs = (uint8_t) *cp++; @@ -1706,6 +1788,14 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) connection_or_close_for_error(chan->conn, 0); return; } + /* A relay can connect from anywhere and be canonical, so + * long as it tells you from where it came. This may be a bit + * concerning.. Luckily we have another check in + * channel_tls_matches_target_method() to ensure that extends + * only go to the IP they ask for. + * + * XXX: Bleh. That check is not used if the connection is canonical. + */ if (tor_addr_eq(&addr, &(chan->conn->real_addr))) { connection_or_set_canonical(chan->conn, 1); break; @@ -1714,6 +1804,21 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) --n_other_addrs; } + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && + channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { + const char *descr = + TLS_CHAN_TO_BASE(chan)->get_remote_descr(TLS_CHAN_TO_BASE(chan), 0); + log_info(LD_OR, + "We made a connection to a relay at %s (fp=%s) but we think " + "they will not consider this connection canonical. They " + "think we are at %s, but we think its %s.", + safe_str(descr), + safe_str(hex_str(chan->conn->identity_digest, DIGEST_LEN)), + safe_str(tor_addr_is_null(&my_apparent_addr) ? + "<none>" : fmt_and_decorate_addr(&my_apparent_addr)), + safe_str(fmt_addr32(me->addr))); + } + /* Act on apparent skew. */ /** Warn when we get a netinfo skew with at least this value. */ #define NETINFO_NOTICE_SKEW 3600 diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index f8b3609757..16cef0e56b 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -816,12 +816,7 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ) * creating on behalf of others. */ return 0; } - if (options->FastFirstHopPK == -1) { - /* option is "auto", so look at the consensus. */ - return networkstatus_get_param(NULL, "usecreatefast", 0, 0, 1); - } - - return options->FastFirstHopPK; + return networkstatus_get_param(NULL, "usecreatefast", 0, 0, 1); } /** Return true if <b>circ</b> is the type of circuit we want to count @@ -939,9 +934,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) memset(&cc, 0, sizeof(cc)); if (circ->build_state->onehop_tunnel) control_event_bootstrap(BOOTSTRAP_STATUS_ONEHOP_CREATE, 0); - else + else { control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0); + /* If this is not a one-hop tunnel, the channel is being used + * for traffic that wants anonymity and protection from traffic + * analysis (such as netflow record retention). That means we want + * to pad it. + */ + if (circ->base_.n_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS) + circ->base_.n_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS; + } + node = node_get_by_id(circ->base_.n_chan->identity_digest); fast = should_use_create_fast_for_circuit(circ); if (!fast) { @@ -1828,15 +1832,16 @@ choose_good_exit_server_general(int need_uptime, int need_capacity) * we'll retry later in this function with need_update and * need_capacity set to 0. */ } - if (!(node->is_valid || options->AllowInvalid_ & ALLOW_INVALID_EXIT)) { + if (!(node->is_valid)) { /* if it's invalid and we don't want it */ n_supported[i] = -1; // log_fn(LOG_DEBUG,"Skipping node %s (index %d) -- invalid router.", // router->nickname, i); continue; /* skip invalid routers */ } - if (options->ExcludeSingleHopRelays && - node_allows_single_hop_exits(node)) { + /* We do not allow relays that allow single hop exits by default. Option + * was deprecated in 0.2.9.2-alpha and removed in 0.3.1.0-alpha. */ + if (node_allows_single_hop_exits(node)) { n_supported[i] = -1; continue; } @@ -1968,7 +1973,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags, const or_options_t *options) { const node_t *rp_node = NULL; - const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; const int need_desc = (flags & CRN_NEED_DESC) != 0; const int pref_addr = (flags & CRN_PREF_ADDR) != 0; const int direct_conn = (flags & CRN_DIRECT_CONN) != 0; @@ -1980,7 +1984,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags, /* Add all running nodes to all_live_nodes */ router_add_running_nodes_to_smartlist(all_live_nodes, - allow_invalid, 0, 0, 0, need_desc, pref_addr, @@ -2022,9 +2025,6 @@ pick_rendezvous_node(router_crn_flags_t flags) { const or_options_t *options = get_options(); - if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS) - flags |= CRN_ALLOW_INVALID; - #ifdef ENABLE_TOR2WEB_MODE /* We want to connect directly to the node if we can */ router_crn_flags_t direct_flags = flags; @@ -2081,8 +2081,6 @@ choose_good_exit_server(uint8_t purpose, switch (purpose) { case CIRCUIT_PURPOSE_C_GENERAL: - if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE) - flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ return router_choose_random_node(NULL, options->ExcludeNodes, flags); else @@ -2280,10 +2278,6 @@ count_acceptable_nodes, (smartlist_t *nodes)) if (! node->is_running) // log_debug(LD_CIRC,"Nope, the directory says %d is not running.",i); continue; - /* XXX This clause makes us count incorrectly: if AllowInvalidRouters - * allows this node in some places, then we're getting an inaccurate - * count. For now, be conservative and don't count it. But later we - * should try to be smarter. */ if (! node->is_valid) // log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i); continue; @@ -2354,8 +2348,6 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_UPTIME; if (state->need_capacity) flags |= CRN_NEED_CAPACITY; - if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE) - flags |= CRN_ALLOW_INVALID; choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; @@ -2408,8 +2400,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state, if (state->need_capacity) flags |= CRN_NEED_CAPACITY; } - if (options->AllowInvalid_ & ALLOW_INVALID_ENTRY) - flags |= CRN_ALLOW_INVALID; choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 80bb7f69f3..5761890924 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -78,6 +78,7 @@ #include "rephist.h" #include "routerlist.h" #include "routerset.h" +#include "channelpadding.h" #include "ht.h" @@ -814,6 +815,11 @@ init_circuit_base(circuit_t *circ) circ->global_circuitlist_idx = smartlist_len(circuit_get_global_list()) - 1; } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idle circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (1*60) + /** Allocate space for a new circuit, initializing with <b>p_circ_id</b> * and <b>p_conn</b>. Add it to the global circuit list. */ @@ -841,6 +847,41 @@ origin_circuit_new(void) circuit_build_times_update_last_circ(get_circuit_build_times_mutable()); + if (! circuit_build_times_disabled(get_options()) && + circuit_build_times_needs_circuits(get_circuit_build_times())) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + circ->circuit_idle_timeout = IDLE_TIMEOUT_WHILE_LEARNING; + } else { + // This should always be larger than the current port prediction time + // remaining, or else we'll end up with the case where a circuit times out + // and another one is built, effectively doubling the timeout window. + // + // We also randomize it by up to 5% more (ie 5% of 0 to 3600 seconds, + // depending on how much circuit prediction time is remaining) so that + // we don't close a bunch of unused circuits all at the same time. + int prediction_time_remaining = + predicted_ports_prediction_time_remaining(time(NULL)); + circ->circuit_idle_timeout = prediction_time_remaining+1+ + crypto_rand_int(1+prediction_time_remaining/20); + + if (circ->circuit_idle_timeout <= 0) { + log_warn(LD_BUG, + "Circuit chose a negative idle timeout of %d based on " + "%d seconds of predictive building remaining.", + circ->circuit_idle_timeout, + prediction_time_remaining); + circ->circuit_idle_timeout = IDLE_TIMEOUT_WHILE_LEARNING; + } + + log_info(LD_CIRC, + "Circuit " U64_FORMAT " chose an idle timeout of %d based on " + "%d seconds of predictive building remaining.", + U64_PRINTF_ARG(circ->global_identifier), + circ->circuit_idle_timeout, + prediction_time_remaining); + } + return circ; } @@ -1990,10 +2031,10 @@ single_conn_free_bytes(connection_t *conn) } if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); - if (dir_conn->zlib_state) { - result += tor_zlib_state_size(dir_conn->zlib_state); - tor_zlib_free(dir_conn->zlib_state); - dir_conn->zlib_state = NULL; + if (dir_conn->compress_state) { + result += tor_compress_state_size(dir_conn->compress_state); + tor_compress_free(dir_conn->compress_state); + dir_conn->compress_state = NULL; } } return result; diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 8d233e0cb6..9f9d3abf7c 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -705,18 +705,15 @@ circuit_expire_building(void) } } - /* If this is a hidden service client circuit which is far enough - * along in connecting to its destination, and we haven't already - * flagged it as 'timed out', and the user has not told us to - * close such circs immediately on timeout, flag it as 'timed out' - * so we'll launch another intro or rend circ, but don't mark it - * for close yet. + /* If this is a hidden service client circuit which is far enough along in + * connecting to its destination, and we haven't already flagged it as + * 'timed out', flag it so we'll launch another intro or rend circ, but + * don't mark it for close yet. * * (Circs flagged as 'timed out' are given a much longer timeout * period above, so we won't close them in the next call to * circuit_expire_building.) */ - if (!(options->CloseHSClientCircuitsImmediatelyOnTimeout) && - !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) { + if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) { switch (victim->purpose) { case CIRCUIT_PURPOSE_C_REND_READY: /* We only want to spare a rend circ if it has been specified in @@ -750,8 +747,7 @@ circuit_expire_building(void) /* If this is a service-side rendezvous circuit which is far * enough along in connecting to its destination, consider sparing * it. */ - if (!(options->CloseHSServiceRendCircuitsImmediatelyOnTimeout) && - !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) && + if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) && victim->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { log_info(LD_CIRC,"Marking circ %u (state %d:%s, purpose %d) " "as timed-out HS circ; relaunching rendezvous attempt.", @@ -1383,11 +1379,6 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } -/** If we haven't yet decided on a good timeout value for circuit - * building, we close idles circuits aggressively so we can get more - * data points. */ -#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) - /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ @@ -1397,21 +1388,15 @@ circuit_expire_old_circuits_clientside(void) struct timeval cutoff, now; tor_gettimeofday(&now); - cutoff = now; last_expired_clientside_circuits = now.tv_sec; - if (! circuit_build_times_disabled(get_options()) && - circuit_build_times_needs_circuits(get_circuit_build_times())) { - /* Circuits should be shorter lived if we need more of them - * for learning a good build timeout */ - cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; - } else { - cutoff.tv_sec -= get_options()->CircuitIdleTimeout; - } - SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; + + cutoff = now; + cutoff.tv_sec -= TO_ORIGIN_CIRCUIT(circ)->circuit_idle_timeout; + /* If the circuit has been dirty for too long, and there are no streams * on it, mark it for close. */ @@ -1437,8 +1422,10 @@ circuit_expire_old_circuits_clientside(void) (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) || circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { - log_debug(LD_CIRC, - "Closing circuit that has been unused for %ld msec.", + log_info(LD_CIRC, + "Closing circuit "U64_FORMAT + " that has been unused for %ld msec.", + U64_PRINTF_ARG(TO_ORIGIN_CIRCUIT(circ)->global_identifier), tv_mdiff(&circ->timestamp_began, &now)); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { diff --git a/src/or/command.c b/src/or/command.c index 0992e97b8b..c667cbbe52 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -326,10 +326,19 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } + if (connection_or_digest_is_known_relay(chan->identity_digest)) { + rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); + // Needed for chutney: Sometimes relays aren't in the consensus yet, and + // get marked as clients. This resets their channels once they appear. + // Probably useful for normal operation wrt relay flapping, too. + chan->is_client = 0; + } else { + channel_mark_client(chan); + } + if (create_cell->handshake_type != ONION_HANDSHAKE_TYPE_FAST) { /* hand it off to the cpuworkers, and then return. */ - if (connection_or_digest_is_known_relay(chan->identity_digest)) - rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); + if (assign_onionskin_to_cpuworker(circ, create_cell) < 0) { log_debug(LD_GENERAL,"Failed to hand off onionskin. Closing."); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); @@ -344,8 +353,14 @@ command_process_create_cell(cell_t *cell, channel_t *chan) int len; created_cell_t created_cell; - /* Make sure we never try to use the OR connection on which we - * received this cell to satisfy an EXTEND request, */ + /* If the client used CREATE_FAST, it's probably a tor client or bridge + * relay, and we must not use it for EXTEND requests (in most cases, we + * won't have an authenticated peer ID for the extend). + * Public relays on 0.2.9 and later will use CREATE_FAST if they have no + * ntor onion key for this relay, but that should be a rare occurrence. + * Clients on 0.3.1 and later avoid using CREATE_FAST as much as they can, + * even during bootstrap, so the CREATE_FAST check is most accurate for + * earlier tor client versions. */ channel_mark_client(chan); memset(&created_cell, 0, sizeof(created_cell)); diff --git a/src/or/config.c b/src/or/config.c index 05c6040480..09d558e1b4 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -69,10 +69,12 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" #include "connection_or.h" +#include "consdiffmgr.h" #include "control.h" #include "confparse.h" #include "cpuworker.h" @@ -99,7 +101,6 @@ #include "statefile.h" #include "transports.h" #include "ext_orport.h" -#include "torgzip.h" #ifdef _WIN32 #include <shlobj.h> #endif @@ -205,10 +206,10 @@ static config_var_t option_vars_[] = { V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), V(AllowDotExit, BOOL, "0"), - V(AllowInvalidNodes, CSV, "middle,rendezvous"), + OBSOLETE("AllowInvalidNodes"), V(AllowNonRFC953Hostnames, BOOL, "0"), - V(AllowSingleHopCircuits, BOOL, "0"), - V(AllowSingleHopExits, BOOL, "0"), + OBSOLETE("AllowSingleHopCircuits"), + OBSOLETE("AllowSingleHopExits"), V(AlternateBridgeAuthority, LINELIST, NULL), V(AlternateDirAuthority, LINELIST, NULL), OBSOLETE("AlternateHSAuthority"), @@ -242,9 +243,11 @@ static config_var_t option_vars_[] = { V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), V(CellStatistics, BOOL, "0"), + V(PaddingStatistics, BOOL, "1"), V(LearnCircuitBuildTimeout, BOOL, "1"), V(CircuitBuildTimeout, INTERVAL, "0"), - V(CircuitIdleTimeout, INTERVAL, "1 hour"), + OBSOLETE("CircuitIdleTimeout"), + V(CircuitsAvailableTimeout, INTERVAL, "0"), V(CircuitStreamTimeout, INTERVAL, "0"), V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), @@ -261,7 +264,7 @@ static config_var_t option_vars_[] = { V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), V(ContactInfo, STRING, NULL), - V(ControlListenAddress, LINELIST, NULL), + OBSOLETE("ControlListenAddress"), VPORT(ControlPort), V(ControlPortFileGroupReadable,BOOL, "0"), V(ControlPortWriteToFile, FILENAME, NULL), @@ -278,7 +281,7 @@ static config_var_t option_vars_[] = { V(DisableNetwork, BOOL, "0"), V(DirAllowPrivateAddresses, BOOL, "0"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"), - V(DirListenAddress, LINELIST, NULL), + OBSOLETE("DirListenAddress"), V(DirPolicy, LINELIST, NULL), VPORT(DirPort), V(DirPortFrontPage, FILENAME, NULL), @@ -292,7 +295,7 @@ static config_var_t option_vars_[] = { OBSOLETE("DisableV2DirectoryInfo_"), OBSOLETE("DynamicDHGroups"), VPORT(DNSPort), - V(DNSListenAddress, LINELIST, NULL), + OBSOLETE("DNSListenAddress"), V(DownloadExtraInfo, BOOL, "0"), V(TestingEnableConnBwEvent, BOOL, "0"), V(TestingEnableCellStatsEvent, BOOL, "0"), @@ -303,7 +306,7 @@ static config_var_t option_vars_[] = { V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), - V(ExcludeSingleHopRelays, BOOL, "1"), + OBSOLETE("ExcludeSingleHopRelays"), V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), @@ -323,7 +326,7 @@ static config_var_t option_vars_[] = { OBSOLETE("FallbackNetworkstatusFile"), V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), - V(FastFirstHopPK, AUTOBOOL, "auto"), + OBSOLETE("FastFirstHopPK"), V(FetchDirInfoEarly, BOOL, "0"), V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), @@ -360,8 +363,8 @@ static config_var_t option_vars_[] = { VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"), V(HidServAuth, LINELIST, NULL), - V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"), - V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"), + OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"), + OBSOLETE("CloseHSServiceRendCircuitsImmediatelyOnTimeout"), V(HiddenServiceSingleHopMode, BOOL, "0"), V(HiddenServiceNonAnonymousMode,BOOL, "0"), V(HTTPProxy, STRING, NULL), @@ -398,17 +401,17 @@ static config_var_t option_vars_[] = { VAR("MyFamily", LINELIST, MyFamily_lines, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), OBSOLETE("NamingAuthoritativeDirectory"), - V(NATDListenAddress, LINELIST, NULL), + OBSOLETE("NATDListenAddress"), VPORT(NATDPort), V(Nickname, STRING, NULL), - V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), - V(WarnUnsafeSocks, BOOL, "1"), + OBSOLETE("PredictedPortsRelevanceTime"), + OBSOLETE("WarnUnsafeSocks"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), V(NumCPUs, UINT, "0"), V(NumDirectoryGuards, UINT, "0"), V(NumEntryGuards, UINT, "0"), V(OfflineMasterKey, BOOL, "0"), - V(ORListenAddress, LINELIST, NULL), + OBSOLETE("ORListenAddress"), VPORT(ORPort), V(OutboundBindAddress, LINELIST, NULL), V(OutboundBindAddressOR, LINELIST, NULL), @@ -458,6 +461,8 @@ static config_var_t option_vars_[] = { V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), V(RecommendedPackages, LINELIST, NULL), + V(ReducedConnectionPadding, BOOL, "0"), + V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), @@ -481,7 +486,7 @@ static config_var_t option_vars_[] = { V(SchedulerHighWaterMark__, MEMUNIT, "101 MB"), V(SchedulerMaxFlushCells__, UINT, "1000"), V(ShutdownWaitLength, INTERVAL, "30 seconds"), - V(SocksListenAddress, LINELIST, NULL), + OBSOLETE("SocksListenAddress"), V(SocksPolicy, LINELIST, NULL), VPORT(SocksPort), V(SocksTimeout, INTERVAL, "2 minutes"), @@ -494,10 +499,10 @@ static config_var_t option_vars_[] = { V(TokenBucketRefillInterval, MSEC_INTERVAL, "100 msec"), V(Tor2webMode, BOOL, "0"), V(Tor2webRendezvousPoints, ROUTERSET, NULL), - V(TLSECGroup, STRING, NULL), + OBSOLETE("TLSECGroup"), V(TrackHostExits, CSV, NULL), V(TrackHostExitsExpire, INTERVAL, "30 minutes"), - V(TransListenAddress, LINELIST, NULL), + OBSOLETE("TransListenAddress"), VPORT(TransPort), V(TransProxyType, STRING, "default"), OBSOLETE("TunnelDirConns"), @@ -662,35 +667,8 @@ static const config_deprecation_t option_deprecation_notes_[] = { /* Deprecated since 0.2.9.2-alpha... */ { "AllowDotExit", "Unrestricted use of the .exit notation can be used for " "a wide variety of application-level attacks." }, - { "AllowInvalidNodes", "There is no reason to enable this option; at best " - "it will make you easier to track." }, - { "AllowSingleHopCircuits", "Almost no relays actually allow single-hop " - "exits, making this option pointless." }, - { "AllowSingleHopExits", "Turning this on will make your relay easier " - "to abuse." }, { "ClientDNSRejectInternalAddresses", "Turning this on makes your client " "easier to fingerprint, and may open you to esoteric attacks." }, - { "ExcludeSingleHopRelays", "Turning it on makes your client easier to " - "fingerprint." }, - { "FastFirstHopPK", "Changing this option does not make your client more " - "secure, but does make it easier to fingerprint." }, - { "CloseHSClientCircuitsImmediatelyOnTimeout", "This option makes your " - "client easier to fingerprint." }, - { "CloseHSServiceRendCircuitsImmediatelyOnTimeout", "This option makes " - "your hidden services easier to fingerprint." }, - { "WarnUnsafeSocks", "Changing this option makes it easier for you " - "to accidentally lose your anonymity by leaking DNS information" }, - { "TLSECGroup", "The default is a nice secure choice; the other option " - "is less secure." }, - { "ControlListenAddress", "Use ControlPort instead." }, - { "DirListenAddress", "Use DirPort instead, possibly with the " - "NoAdvertise sub-option" }, - { "DNSListenAddress", "Use DNSPort instead." }, - { "SocksListenAddress", "Use SocksPort instead." }, - { "TransListenAddress", "Use TransPort instead." }, - { "NATDListenAddress", "Use NATDPort instead." }, - { "ORListenAddress", "Use ORPort instead, possibly with the " - "NoAdvertise sub-option" }, /* End of options deprecated since 0.2.9.2-alpha. */ { NULL, NULL } @@ -1556,23 +1534,6 @@ get_effective_bwburst(const or_options_t *options) return (uint32_t)bw; } -/** Return True if any changes from <b>old_options</b> to - * <b>new_options</b> needs us to refresh our TLS context. */ -static int -options_transition_requires_fresh_tls_context(const or_options_t *old_options, - const or_options_t *new_options) -{ - tor_assert(new_options); - - if (!old_options) - return 0; - - if (!opt_streq(old_options->TLSECGroup, new_options->TLSECGroup)) - return 1; - - return 0; -} - /** * Return true if changing the configuration from <b>old</b> to <b>new</b> * affects the guard susbsystem. @@ -1791,13 +1752,6 @@ options_act(const or_options_t *old_options) log_warn(LD_BUG,"Error initializing keys; exiting"); return -1; } - } else if (old_options && - options_transition_requires_fresh_tls_context(old_options, - options)) { - if (router_initialize_tls_context() < 0) { - log_warn(LD_BUG,"Error initializing TLS context."); - return -1; - } } /* Write our PID to the PID file. If we do not have write permissions we @@ -1818,6 +1772,15 @@ options_act(const or_options_t *old_options) return -1; } + if (server_mode(options)) { + static int cdm_initialized = 0; + if (cdm_initialized == 0) { + cdm_initialized = 1; + consdiffmgr_configure(NULL); + consdiffmgr_validate(); + } + } + if (init_control_cookie_authentication(options->CookieAuthentication) < 0) { log_warn(LD_CONFIG,"Error creating control cookie authentication file."); return -1; @@ -2812,10 +2775,10 @@ compute_publishserverdescriptor(or_options_t *options) #define MIN_REND_POST_PERIOD (10*60) #define MIN_REND_POST_PERIOD_TESTING (5) -/** Highest allowable value for PredictedPortsRelevanceTime; if this is - * too high, our selection of exits will decrease for an extended - * period of time to an uncomfortable level .*/ -#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) +/** Higest allowable value for CircuitsAvailableTimeout. + * If this is too large, client connections will stay open for too long, + * incurring extra padding overhead. */ +#define MAX_CIRCS_AVAILABLE_TIME (24*60*60) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD ((7*24*60*60)/2) @@ -3001,6 +2964,10 @@ options_validate(or_options_t *old_options, or_options_t *options, tor_assert(msg); *msg = NULL; + if (parse_ports(options, 1, msg, &n_ports, + &world_writable_control_socket) < 0) + return -1; + /* Set UseEntryGuards from the configured value, before we check it below. * We change UseEntryGuards when it's incompatible with other options, * but leave UseEntryGuards_option with the original value. @@ -3019,10 +2986,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "for details.", uname); } - if (parse_ports(options, 1, msg, &n_ports, - &world_writable_control_socket) < 0) - return -1; - if (parse_outbound_addresses(options, 1, msg) < 0) return -1; @@ -3115,14 +3078,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (strcasecmp(options->TransProxyType, "default") && !options->TransPort_set) { - REJECT("Cannot use TransProxyType without any valid TransPort or " - "TransListenAddress."); + REJECT("Cannot use TransProxyType without any valid TransPort."); } } #else if (options->TransPort_set) - REJECT("TransPort and TransListenAddress are disabled " - "in this build."); + REJECT("TransPort is disabled in this build."); #endif if (options->TokenBucketRefillInterval <= 0 @@ -3159,15 +3120,6 @@ options_validate(or_options_t *old_options, or_options_t *options, } } - if (options->TLSECGroup && (strcasecmp(options->TLSECGroup, "P256") && - strcasecmp(options->TLSECGroup, "P224"))) { - COMPLAIN("Unrecognized TLSECGroup: Falling back to the default."); - tor_free(options->TLSECGroup); - } - if (!evaluate_ecgroup_for_tls(options->TLSECGroup)) { - REJECT("Unsupported TLSECGroup."); - } - if (options->ExcludeNodes && options->StrictNodes) { COMPLAIN("You have asked to exclude certain relays from all positions " "in your circuits. Expect hidden services and other Tor " @@ -3375,28 +3327,6 @@ options_validate(or_options_t *old_options, or_options_t *options, server_mode(options)); options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3; - options->AllowInvalid_ = 0; - - if (options->AllowInvalidNodes) { - SMARTLIST_FOREACH_BEGIN(options->AllowInvalidNodes, const char *, cp) { - if (!strcasecmp(cp, "entry")) - options->AllowInvalid_ |= ALLOW_INVALID_ENTRY; - else if (!strcasecmp(cp, "exit")) - options->AllowInvalid_ |= ALLOW_INVALID_EXIT; - else if (!strcasecmp(cp, "middle")) - options->AllowInvalid_ |= ALLOW_INVALID_MIDDLE; - else if (!strcasecmp(cp, "introduction")) - options->AllowInvalid_ |= ALLOW_INVALID_INTRODUCTION; - else if (!strcasecmp(cp, "rendezvous")) - options->AllowInvalid_ |= ALLOW_INVALID_RENDEZVOUS; - else { - tor_asprintf(msg, - "Unrecognized value '%s' in AllowInvalidNodes", cp); - return -1; - } - } SMARTLIST_FOREACH_END(cp); - } - if (!options->SafeLogging || !strcasecmp(options->SafeLogging, "0")) { options->SafeLogging_ = SAFELOG_SCRUB_NONE; @@ -3432,6 +3362,14 @@ options_validate(or_options_t *old_options, or_options_t *options, options->DirPort_set = 0; } + if (server_mode(options) && options->ConnectionPadding != -1) { + REJECT("Relays must use 'auto' for the ConnectionPadding setting."); + } + + if (server_mode(options) && options->ReducedConnectionPadding != 0) { + REJECT("Relays cannot set ReducedConnectionPadding. "); + } + if (options->MinUptimeHidServDirectoryV2 < 0) { log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at " "least 0 seconds. Changing to 0."); @@ -3453,17 +3391,17 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->PredictedPortsRelevanceTime > - MAX_PREDICTED_CIRCS_RELEVANCE) { - log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; " - "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE); - options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; - } - /* Check the Single Onion Service options */ if (options_validate_single_onion(options, msg) < 0) return -1; + if (options->CircuitsAvailableTimeout > MAX_CIRCS_AVAILABLE_TIME) { + // options_t is immutable for new code (the above code is older), + // so just make the user fix the value themselves rather than + // silently keep a shadow value lower than what they asked for. + REJECT("CircuitsAvailableTimeout is too large. Max is 24 hours."); + } + #ifdef ENABLE_TOR2WEB_MODE if (options->Tor2webMode && options->UseEntryGuards) { /* tor2web mode clients do not (and should not) use entry guards @@ -4086,13 +4024,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "AlternateDirAuthority and AlternateBridgeAuthority configured."); } - if (options->AllowSingleHopExits && !options->DirAuthorities) { - COMPLAIN("You have set AllowSingleHopExits; now your relay will allow " - "others to make one-hop exits. However, since by default most " - "clients avoid relays that set this option, most clients will " - "ignore you."); - } - #define CHECK_DEFAULT(arg) \ STMT_BEGIN \ if (!options->TestingTorNetwork && \ @@ -4971,9 +4902,21 @@ options_init_from_torrc(int argc, char **argv) printf("OpenSSL \t\t%-15s\t\t%s\n", crypto_openssl_get_header_version_str(), crypto_openssl_get_version_str()); - printf("Zlib \t\t%-15s\t\t%s\n", - tor_zlib_get_header_version_str(), - tor_zlib_get_version_str()); + if (tor_compress_supports_method(ZLIB_METHOD)) { + printf("Zlib \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZLIB_METHOD), + tor_compress_header_version_str(ZLIB_METHOD)); + } + if (tor_compress_supports_method(LZMA_METHOD)) { + printf("Liblzma \t\t%-15s\t\t%s\n", + tor_compress_version_str(LZMA_METHOD), + tor_compress_header_version_str(LZMA_METHOD)); + } + if (tor_compress_supports_method(ZSTD_METHOD)) { + printf("Libzstd \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZSTD_METHOD), + tor_compress_header_version_str(ZSTD_METHOD)); + } //TODO: Hex versions? exit(0); } @@ -6459,14 +6402,9 @@ warn_client_dns_cache(const char *option, int disabling) /** * Parse port configuration for a single port type. * - * Read entries of the "FooPort" type from the list <b>ports</b>, and - * entries of the "FooListenAddress" type from the list - * <b>listenaddrs</b>. Two syntaxes are supported: a legacy syntax - * where FooPort is at most a single entry containing a port number and - * where FooListenAddress has any number of address:port combinations; - * and a new syntax where there are no FooListenAddress entries and - * where FooPort can have any number of entries of the format - * "[Address:][Port] IsolationOptions". + * Read entries of the "FooPort" type from the list <b>ports</b>. Syntax is + * that FooPort can have any number of entries of the format + * "[Address:][Port] IsolationOptions". * * In log messages, describe the port type as <b>portname</b>. * @@ -6480,9 +6418,6 @@ warn_client_dns_cache(const char *option, int disabling) * ports are not on a local address. If CL_PORT_FORBID_NONLOCAL is set, * this is a control port with no password set: don't even allow it. * - * Unless CL_PORT_ALLOW_EXTRA_LISTENADDR is set in <b>flags</b>, warn - * if FooListenAddress is set but FooPort is 0. - * * If CL_PORT_SERVER_OPTIONS is set in <b>flags</b>, do not allow stream * isolation options in the FooPort entries; instead allow the * server-port option set. @@ -6497,7 +6432,6 @@ warn_client_dns_cache(const char *option, int disabling) STATIC int parse_port_config(smartlist_t *out, const config_line_t *ports, - const config_line_t *listenaddrs, const char *portname, int listener_type, const char *defaultaddr, @@ -6514,90 +6448,12 @@ parse_port_config(smartlist_t *out, const unsigned forbid_nonlocal = flags & CL_PORT_FORBID_NONLOCAL; const unsigned default_to_group_writable = flags & CL_PORT_DFLT_GROUP_WRITABLE; - const unsigned allow_spurious_listenaddr = - flags & CL_PORT_ALLOW_EXTRA_LISTENADDR; const unsigned takes_hostnames = flags & CL_PORT_TAKES_HOSTNAMES; const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET; int got_zero_port=0, got_nonzero_port=0; char *unix_socket_path = NULL; - /* FooListenAddress is deprecated; let's make it work like it used to work, - * though. */ - if (listenaddrs) { - int mainport = defaultport; - - if (ports && ports->next) { - log_warn(LD_CONFIG, "%sListenAddress can't be used when there are " - "multiple %sPort lines", portname, portname); - return -1; - } else if (ports) { - if (!strcmp(ports->value, "auto")) { - mainport = CFG_AUTO_PORT; - } else { - int ok; - mainport = (int)tor_parse_long(ports->value, 10, 0, 65535, &ok, NULL); - if (!ok) { - log_warn(LD_CONFIG, "%sListenAddress can only be used with a single " - "%sPort with value \"auto\" or 1-65535 and no options set.", - portname, portname); - return -1; - } - } - } - - if (mainport == 0) { - if (allow_spurious_listenaddr) - return 1; /*DOCDOC*/ - log_warn(LD_CONFIG, "%sPort must be defined if %sListenAddress is used", - portname, portname); - return -1; - } - - if (use_server_options && out) { - /* Add a no_listen port. */ - port_cfg_t *cfg = port_cfg_new(0); - cfg->type = listener_type; - cfg->port = mainport; - tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */ - cfg->server_cfg.no_listen = 1; - cfg->server_cfg.bind_ipv4_only = 1; - /* cfg->entry_cfg defaults are already set by port_cfg_new */ - smartlist_add(out, cfg); - } - - for (; listenaddrs; listenaddrs = listenaddrs->next) { - tor_addr_t addr; - uint16_t port = 0; - if (tor_addr_port_lookup(listenaddrs->value, &addr, &port) < 0) { - log_warn(LD_CONFIG, "Unable to parse %sListenAddress '%s'", - portname, listenaddrs->value); - return -1; - } - if (out) { - port_cfg_t *cfg = port_cfg_new(0); - cfg->type = listener_type; - cfg->port = port ? port : mainport; - tor_addr_copy(&cfg->addr, &addr); - cfg->entry_cfg.session_group = SESSION_GROUP_UNSET; - cfg->entry_cfg.isolation_flags = ISO_DEFAULT; - cfg->server_cfg.no_advertise = 1; - smartlist_add(out, cfg); - } - } - - if (warn_nonlocal && out) { - if (is_control) - warn_nonlocal_controller_ports(out, forbid_nonlocal); - else if (is_ext_orport) - warn_nonlocal_ext_orports(out, portname); - else - warn_nonlocal_client_ports(out, portname, listener_type); - } - return 0; - } /* end if (listenaddrs) */ - - /* No ListenAddress lines. If there's no FooPort, then maybe make a default - * one. */ + /* If there's no FooPort, then maybe make a default one. */ if (! ports) { if (defaultport && defaultaddr && out) { port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0); @@ -7068,36 +6924,35 @@ parse_ports(or_options_t *options, int validate_only, const unsigned gw_flag = options->SocksSocketsGroupWritable ? CL_PORT_DFLT_GROUP_WRITABLE : 0; if (parse_port_config(ports, - options->SocksPort_lines, options->SocksListenAddress, + options->SocksPort_lines, "Socks", CONN_TYPE_AP_LISTENER, "127.0.0.1", 9050, - CL_PORT_WARN_NONLOCAL|CL_PORT_ALLOW_EXTRA_LISTENADDR| - CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) { - *msg = tor_strdup("Invalid SocksPort/SocksListenAddress configuration"); + CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) { + *msg = tor_strdup("Invalid SocksPort configuration"); goto err; } if (parse_port_config(ports, - options->DNSPort_lines, options->DNSListenAddress, + options->DNSPort_lines, "DNS", CONN_TYPE_AP_DNS_LISTENER, "127.0.0.1", 0, CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES) < 0) { - *msg = tor_strdup("Invalid DNSPort/DNSListenAddress configuration"); + *msg = tor_strdup("Invalid DNSPort configuration"); goto err; } if (parse_port_config(ports, - options->TransPort_lines, options->TransListenAddress, + options->TransPort_lines, "Trans", CONN_TYPE_AP_TRANS_LISTENER, "127.0.0.1", 0, CL_PORT_WARN_NONLOCAL) < 0) { - *msg = tor_strdup("Invalid TransPort/TransListenAddress configuration"); + *msg = tor_strdup("Invalid TransPort configuration"); goto err; } if (parse_port_config(ports, - options->NATDPort_lines, options->NATDListenAddress, + options->NATDPort_lines, "NATD", CONN_TYPE_AP_NATD_LISTENER, "127.0.0.1", 0, CL_PORT_WARN_NONLOCAL) < 0) { - *msg = tor_strdup("Invalid NatdPort/NatdListenAddress configuration"); + *msg = tor_strdup("Invalid NatdPort configuration"); goto err; } { @@ -7113,16 +6968,14 @@ parse_ports(or_options_t *options, int validate_only, if (parse_port_config(ports, options->ControlPort_lines, - options->ControlListenAddress, "Control", CONN_TYPE_CONTROL_LISTENER, "127.0.0.1", 0, control_port_flags) < 0) { - *msg = tor_strdup("Invalid ControlPort/ControlListenAddress " - "configuration"); + *msg = tor_strdup("Invalid ControlPort configuration"); goto err; } - if (parse_port_config(ports, options->ControlSocket, NULL, + if (parse_port_config(ports, options->ControlSocket, "ControlSocket", CONN_TYPE_CONTROL_LISTENER, NULL, 0, control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) { @@ -7132,15 +6985,15 @@ parse_ports(or_options_t *options, int validate_only, } if (! options->ClientOnly) { if (parse_port_config(ports, - options->ORPort_lines, options->ORListenAddress, + options->ORPort_lines, "OR", CONN_TYPE_OR_LISTENER, "0.0.0.0", 0, CL_PORT_SERVER_OPTIONS) < 0) { - *msg = tor_strdup("Invalid ORPort/ORListenAddress configuration"); + *msg = tor_strdup("Invalid ORPort configuration"); goto err; } if (parse_port_config(ports, - options->ExtORPort_lines, NULL, + options->ExtORPort_lines, "ExtOR", CONN_TYPE_EXT_OR_LISTENER, "127.0.0.1", 0, CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) { @@ -7148,11 +7001,11 @@ parse_ports(or_options_t *options, int validate_only, goto err; } if (parse_port_config(ports, - options->DirPort_lines, options->DirListenAddress, + options->DirPort_lines, "Dir", CONN_TYPE_DIR_LISTENER, "0.0.0.0", 0, CL_PORT_SERVER_OPTIONS) < 0) { - *msg = tor_strdup("Invalid DirPort/DirListenAddress configuration"); + *msg = tor_strdup("Invalid DirPort configuration"); goto err; } } diff --git a/src/or/config.h b/src/or/config.h index bb7802c990..27aec7fe3d 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -157,7 +157,7 @@ smartlist_t *get_options_for_server_transport(const char *transport); #define CL_PORT_NO_STREAM_OPTIONS (1u<<0) #define CL_PORT_WARN_NONLOCAL (1u<<1) -#define CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) +/* Was CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) */ #define CL_PORT_SERVER_OPTIONS (1u<<3) #define CL_PORT_FORBID_NONLOCAL (1u<<4) #define CL_PORT_TAKES_HOSTNAMES (1u<<5) @@ -193,7 +193,6 @@ STATIC int have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, char **msg); STATIC int parse_port_config(smartlist_t *out, const config_line_t *ports, - const config_line_t *listenaddrs, const char *portname, int listener_type, const char *defaultaddr, diff --git a/src/or/confparse.c b/src/or/confparse.c index 75ec92e30f..abae7e33dc 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -1190,8 +1190,6 @@ config_parse_msec_interval(const char *s, int *ok) { uint64_t r; r = config_parse_units(s, time_msec_units, ok); - if (!ok) - return -1; if (r > INT_MAX) { log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); *ok = 0; @@ -1209,8 +1207,6 @@ config_parse_interval(const char *s, int *ok) { uint64_t r; r = config_parse_units(s, time_units, ok); - if (!ok) - return -1; if (r > INT_MAX) { log_warn(LD_CONFIG, "Interval '%s' is too long", s); *ok = 0; diff --git a/src/or/connection.c b/src/or/connection.c index 09e316d214..5fb2c53677 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -628,7 +628,7 @@ connection_free_(connection_t *conn) dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - tor_zlib_free(dir_conn->zlib_state); + tor_compress_free(dir_conn->compress_state); if (dir_conn->spool) { SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled, spooled_resource_free(spooled)); @@ -4060,9 +4060,9 @@ connection_write_to_buf_impl_,(const char *string, size_t len, if (zlib) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); int done = zlib < 0; - CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf, - dir_conn->zlib_state, - string, len, done)); + CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf, + dir_conn->compress_state, + string, len, done)); } else { CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf)); } diff --git a/src/or/connection.h b/src/or/connection.h index df6fc64709..36e45aef38 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -141,17 +141,17 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); -/* DOCDOC connection_write_to_buf_zlib */ -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); +/* DOCDOC connection_write_to_buf_compress */ +static void connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done); static inline void connection_write_to_buf(const char *string, size_t len, connection_t *conn) { connection_write_to_buf_impl_(string, len, conn, 0); } static inline void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) +connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done) { connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1); } diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 2c60d8dddb..6713ffa86f 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2527,7 +2527,7 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn) /* Sensitive directory connections must have an anonymous path length. * Otherwise, directory connections are typically one-hop. * This matches the earlier check for directory connection path anonymity - * in directory_initiate_command_rend(). */ + * in directory_initiate_request(). */ if (purpose_needs_anonymity(linked_dir_conn_base->purpose, TO_DIR_CONN(linked_dir_conn_base)->router_purpose, TO_DIR_CONN(linked_dir_conn_base)->requested_resource)) { @@ -3133,15 +3133,13 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) port = bcell.port; if (or_circ && or_circ->p_chan) { - if (!options->AllowSingleHopExits && - (or_circ->is_first_hop || - (!connection_or_digest_is_known_relay( + if ((or_circ->is_first_hop || + (!connection_or_digest_is_known_relay( or_circ->p_chan->identity_digest) && should_refuse_unknown_exits(options)))) { - /* Don't let clients use us as a single-hop proxy, unless the user - * has explicitly allowed that in the config. It attracts attackers - * and users who'd be better off with, well, single-hop proxies. - */ + /* Don't let clients use us as a single-hop proxy. It attracts + * attackers and users who'd be better off with, well, single-hop + * proxies. */ log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Attempt by %s to open a stream %s. Closing.", safe_str(channel_get_canonical_remote_descr(or_circ->p_chan)), diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 61da43e119..280f8f70ad 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -55,6 +55,7 @@ #include "ext_orport.h" #include "scheduler.h" #include "torcert.h" +#include "channelpadding.h" static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_launch_v3_or_handshake(or_connection_t *conn); @@ -814,24 +815,6 @@ connection_or_update_token_buckets(smartlist_t *conns, }); } -/** How long do we wait before killing non-canonical OR connections with no - * circuits? In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15 - * minutes before cancelling these connections, which caused fast relays to - * accrue many many idle connections. Hopefully 3-4.5 minutes is low enough - * that it kills most idle connections, without being so low that we cause - * clients to bounce on and off. - * - * For canonical connections, the limit is higher, at 15-22.5 minutes. - * - * For each OR connection, we randomly add up to 50% extra to its idle_timeout - * field, to avoid exposing when exactly the last circuit closed. Since we're - * storing idle_timeout in a uint16_t, don't let these values get higher than - * 12 hours or so without revising connection_or_set_canonical and/or expanding - * idle_timeout. - */ -#define IDLE_OR_CONN_TIMEOUT_NONCANONICAL 180 -#define IDLE_OR_CONN_TIMEOUT_CANONICAL 900 - /* Mark <b>or_conn</b> as canonical if <b>is_canonical</b> is set, and * non-canonical otherwise. Adjust idle_timeout accordingly. */ @@ -839,9 +822,6 @@ void connection_or_set_canonical(or_connection_t *or_conn, int is_canonical) { - const unsigned int timeout_base = is_canonical ? - IDLE_OR_CONN_TIMEOUT_CANONICAL : IDLE_OR_CONN_TIMEOUT_NONCANONICAL; - if (bool_eq(is_canonical, or_conn->is_canonical) && or_conn->idle_timeout != 0) { /* Don't recalculate an existing idle_timeout unless the canonical @@ -850,7 +830,14 @@ connection_or_set_canonical(or_connection_t *or_conn, } or_conn->is_canonical = !! is_canonical; /* force to a 1-bit boolean */ - or_conn->idle_timeout = timeout_base + crypto_rand_int(timeout_base / 2); + or_conn->idle_timeout = channelpadding_get_channel_idle_timeout( + TLS_CHAN_TO_BASE(or_conn->chan), is_canonical); + + log_info(LD_CIRC, + "Channel " U64_FORMAT " chose an idle timeout of %d.", + or_conn->chan ? + U64_PRINTF_ARG(TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier):0, + or_conn->idle_timeout); } /** If we don't necessarily know the router we're connecting to, but we @@ -1053,10 +1040,8 @@ connection_or_group_set_badness_(smartlist_t *group, int force) } if (!best || - channel_is_better(now, - TLS_CHAN_TO_BASE(or_conn->chan), - TLS_CHAN_TO_BASE(best->chan), - 0)) { + channel_is_better(TLS_CHAN_TO_BASE(or_conn->chan), + TLS_CHAN_TO_BASE(best->chan))) { best = or_conn; } } SMARTLIST_FOREACH_END(or_conn); @@ -1084,11 +1069,9 @@ connection_or_group_set_badness_(smartlist_t *group, int force) or_conn->base_.state != OR_CONN_STATE_OPEN) continue; if (or_conn != best && - channel_is_better(now, - TLS_CHAN_TO_BASE(best->chan), - TLS_CHAN_TO_BASE(or_conn->chan), 1)) { - /* This isn't the best conn, _and_ the best conn is better than it, - even when we're being forgiving. */ + channel_is_better(TLS_CHAN_TO_BASE(best->chan), + TLS_CHAN_TO_BASE(or_conn->chan))) { + /* This isn't the best conn, _and_ the best conn is better than it */ if (best->is_canonical) { log_info(LD_OR, "Marking OR conn to %s:%d as unsuitable for new circuits: " @@ -1983,12 +1966,23 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn) cell_pack(&networkcell, cell, conn->wide_circ_ids); + rep_hist_padding_count_write(PADDING_TYPE_TOTAL); + if (cell->command == CELL_PADDING) + rep_hist_padding_count_write(PADDING_TYPE_CELL); + connection_write_to_buf(networkcell.body, cell_network_size, TO_CONN(conn)); /* Touch the channel's active timestamp if there is one */ - if (conn->chan) + if (conn->chan) { channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); + if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) { + rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL); + if (cell->command == CELL_PADDING) + rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL); + } + } + if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_cell(conn, conn->handshake_state, cell, 0); } @@ -2094,7 +2088,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) } /** Array of recognized link protocol versions. */ -static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4 }; +static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4, 5 }; /** Number of versions in <b>or_protocol_versions</b>. */ static const int n_or_protocol_versions = (int)( sizeof(or_protocol_versions)/sizeof(uint16_t) ); diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 40008426e9..4261658932 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -109,6 +109,8 @@ void var_cell_free(var_cell_t *cell); /* DOCDOC */ #define MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS 4 +#define MIN_LINK_PROTO_FOR_CHANNEL_PADDING 5 +#define MAX_LINK_PROTO MIN_LINK_PROTO_FOR_CHANNEL_PADDING void connection_or_group_set_badness_(smartlist_t *group, int force); diff --git a/src/or/conscache.c b/src/or/conscache.c index 2a6e1445e3..5ffa129bbe 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -16,6 +16,7 @@ */ struct consensus_cache_entry_t { uint32_t magic; /**< Must be set to CCE_MAGIC */ + HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t); int32_t refcnt; /**< Reference count. */ unsigned can_remove : 1; /**< If true, we want to delete this file. */ /** If true, we intend to unmap this file as soon as we're done with it. */ @@ -77,13 +78,24 @@ consensus_cache_open(const char *subdir, int max_entries) } /** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that <b>cache</b> will need. + */ +int +consensus_cache_register_with_sandbox(consensus_cache_t *cache, + struct sandbox_cfg_elem **cfg) +{ + return storage_dir_register_with_sandbox(cache->dir, cfg); +} + +/** * Helper: clear all entries from <b>cache</b> (but do not delete * any that aren't marked for removal */ static void consensus_cache_clear(consensus_cache_t *cache) { - consensus_cache_delete_pending(cache); + consensus_cache_delete_pending(cache, 0); SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { ent->in_cache = NULL; @@ -174,6 +186,8 @@ consensus_cache_find_first(consensus_cache_t *cache, * Given a <b>cache</b>, add every entry to <b>out<b> for which * <b>key</b>=<b>value</b>. If <b>key</b> is NULL, add every entry. * + * Do not add any entry that has been marked for removal. + * * Does not adjust reference counts. */ void @@ -182,12 +196,15 @@ consensus_cache_find_all(smartlist_t *out, const char *key, const char *value) { - if (! key) { - smartlist_add_all(out, cache->entries); - return; - } - SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { + if (ent->can_remove == 1) { + /* We want to delete this; pretend it isn't there. */ + continue; + } + if (! key) { + smartlist_add(out, ent); + continue; + } const char *found_val = consensus_cache_entry_get_value(ent, key); if (found_val && !strcmp(value, found_val)) { smartlist_add(out, ent); @@ -300,6 +317,7 @@ consensus_cache_entry_decref(consensus_cache_entry_t *ent) } tor_free(ent->fname); config_free_lines(ent->labels); + consensus_cache_entry_handles_clear(ent); memwipe(ent, 0, sizeof(consensus_cache_entry_t)); tor_free(ent); } @@ -382,18 +400,33 @@ consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff) } /** + * Return the number of currently unused filenames available in this cache. + */ +int +consensus_cache_get_n_filenames_available(consensus_cache_t *cache) +{ + tor_assert(cache); + int max = storage_dir_get_max_files(cache->dir); + int used = smartlist_len(storage_dir_list(cache->dir)); + tor_assert_nonfatal(max >= used); + return max - used; +} + +/** * Delete every element of <b>cache</b> has been marked with - * consensus_cache_entry_mark_for_removal, and which is not in use except by - * the cache. + * consensus_cache_entry_mark_for_removal. If <b>force</b> is false, + * retain those entries which are not in use except by the cache. */ void -consensus_cache_delete_pending(consensus_cache_t *cache) +consensus_cache_delete_pending(consensus_cache_t *cache, int force) { SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { tor_assert_nonfatal(ent->in_cache == cache); - if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) { - /* Somebody is using this entry right now */ - continue; + if (! force) { + if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) { + /* Somebody is using this entry right now */ + continue; + } } if (ent->can_remove == 0) { /* Don't want to delete this. */ @@ -485,6 +518,8 @@ consensus_cache_entry_unmap(consensus_cache_entry_t *ent) ent->unused_since = TIME_MAX; } +HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, ) + #ifdef TOR_UNIT_TESTS /** * Testing only: Return true iff <b>ent</b> is mapped into memory. diff --git a/src/or/conscache.h b/src/or/conscache.h index 94d7f15457..aef54201f0 100644 --- a/src/or/conscache.h +++ b/src/or/conscache.h @@ -4,13 +4,22 @@ #ifndef TOR_CONSCACHE_H #define TOR_CONSCACHE_H +#include "handles.h" + typedef struct consensus_cache_entry_t consensus_cache_entry_t; typedef struct consensus_cache_t consensus_cache_t; +HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, ) + consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries); void consensus_cache_free(consensus_cache_t *cache); +struct sandbox_cfg_elem; +int consensus_cache_register_with_sandbox(consensus_cache_t *cache, + struct sandbox_cfg_elem **cfg); void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff); -void consensus_cache_delete_pending(consensus_cache_t *cache); +void consensus_cache_delete_pending(consensus_cache_t *cache, + int force); +int consensus_cache_get_n_filenames_available(consensus_cache_t *cache); consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache, const config_line_t *labels, const uint8_t *data, diff --git a/src/or/consdiff.c b/src/or/consdiff.c index d2a2af1b5f..1baa11897c 100644 --- a/src/or/consdiff.c +++ b/src/or/consdiff.c @@ -65,17 +65,35 @@ line_str_eq(const cdline_t *a, const char *b) return lines_eq(a, &bline); } -/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated - * string s. Use the provided memory area for storage. */ -STATIC void -smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) +/** Return true iff a begins with the same contents as the nul-terminated + * string b. */ +static int +line_starts_with_str(const cdline_t *a, const char *b) +{ + const size_t len = strlen(b); + tor_assert(len <= UINT32_MAX); + return a->len >= len && fast_memeq(a->s, b, len); +} + +/** Return a new cdline_t holding as its contents the nul-terminated + * string s. Use the provided memory area for storage. */ +static cdline_t * +cdline_linecpy(memarea_t *area, const char *s) { size_t len = strlen(s); const char *ss = memarea_memdup(area, s, len); cdline_t *line = memarea_alloc(area, sizeof(cdline_t)); line->s = ss; line->len = (uint32_t)len; - smartlist_add(lst, line); + return line; +} + +/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated + * string s. Use the provided memory area for storage. */ +STATIC void +smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) +{ + smartlist_add(lst, cdline_linecpy(area, s)); } /** Compute the digest of <b>cons</b>, and store the result in @@ -91,6 +109,18 @@ consensus_compute_digest,(const char *cons, return r; } +/** Compute the digest-as-signed of <b>cons</b>, and store the result in + * <b>digest_out</b>. Return 0 on success, -1 on failure. */ +/* This is a separate, mockable function so that we can override it when + * fuzzing. */ +MOCK_IMPL(STATIC int, +consensus_compute_digest_as_signed,(const char *cons, + consensus_digest_t *digest_out)) +{ + return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256, + cons); +} + /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */ /* This is a separate, mockable function so that we can override it when * fuzzing. */ @@ -533,6 +563,42 @@ find_next_router_line(const smartlist_t *cons, return 0; } +/** Line-prefix indicating the beginning of the signatures section that we + * intend to delete. */ +#define START_OF_SIGNATURES_SECTION "directory-signature " + +/** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t) + * to remove the signatures from it. If the footer is removed, return a + * cdline_t containing a delete command to delete the footer, allocated in + * <b>area</>. If no footer is removed, return NULL. + * + * We remove the signatures here because they are not themselves signed, and + * as such there might be different encodings for them. + */ +static cdline_t * +preprocess_consensus(memarea_t *area, + smartlist_t *cons) +{ + int idx; + int dirsig_idx = -1; + for (idx = 0; idx < smartlist_len(cons); ++idx) { + cdline_t *line = smartlist_get(cons, idx); + if (line_starts_with_str(line, START_OF_SIGNATURES_SECTION)) { + dirsig_idx = idx; + break; + } + } + if (dirsig_idx >= 0) { + char buf[64]; + while (smartlist_len(cons) > dirsig_idx) + smartlist_del(cons, dirsig_idx); + tor_snprintf(buf, sizeof(buf), "%d,$d", dirsig_idx+1); + return cdline_linecpy(area, buf); + } else { + return NULL; + } +} + /** Generate an ed diff as a smartlist from two consensuses, also given as * smartlists. Will return NULL if the diff could not be generated, which can * happen if any lines the script had to add matched "." or if the routers @@ -554,13 +620,22 @@ find_next_router_line(const smartlist_t *cons, * calc_changes(cons1_sl, cons2_sl, changed1, changed2); */ STATIC smartlist_t * -gen_ed_diff(const smartlist_t *cons1, const smartlist_t *cons2, +gen_ed_diff(const smartlist_t *cons1_orig, const smartlist_t *cons2, memarea_t *area) { + smartlist_t *cons1 = smartlist_new(); + smartlist_add_all(cons1, cons1_orig); + cdline_t *remove_trailer = preprocess_consensus(area, cons1); + int len1 = smartlist_len(cons1); int len2 = smartlist_len(cons2); smartlist_t *result = smartlist_new(); + if (remove_trailer) { + /* There's a delete-the-trailer line at the end, so add it here. */ + smartlist_add(result, remove_trailer); + } + /* Initialize the changed bitarrays to zero, so that calc_changes only needs * to set the ones that matter and leave the rest untouched. */ @@ -671,7 +746,7 @@ gen_ed_diff(const smartlist_t *cons1, const smartlist_t *cons2, */ i1=len1-1, i2=len2-1; char buf[128]; - while (i1 > 0 || i2 > 0) { + while (i1 >= 0 || i2 >= 0) { int start1x, start2x, end1, end2, added, deleted; @@ -734,6 +809,7 @@ gen_ed_diff(const smartlist_t *cons1, const smartlist_t *cons2, } } + smartlist_free(cons1); bitarray_free(changed1); bitarray_free(changed2); @@ -741,6 +817,7 @@ gen_ed_diff(const smartlist_t *cons1, const smartlist_t *cons2, error_cleanup: + smartlist_free(cons1); bitarray_free(changed1); bitarray_free(changed2); @@ -757,6 +834,9 @@ get_linenum(const char **s, int *num_out) { int ok; char *next; + if (!TOR_ISDIGIT(**s)) { + return -1; + } *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next); if (ok && next) { *s = next; @@ -795,6 +875,8 @@ apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, diff_line[diff_cdline->len] = 0; const char *ptr = diff_line; int start = 0, end = 0; + int had_range = 0; + int end_was_eof = 0; if (get_linenum(&ptr, &start) < 0) { log_warn(LD_CONSDIFF, "Could not apply consensus diff because " "an ed command was missing a line number."); @@ -802,8 +884,13 @@ apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, } if (*ptr == ',') { /* Two-item range */ + had_range = 1; ++ptr; - if (get_linenum(&ptr, &end) < 0) { + if (*ptr == '$') { + end_was_eof = 1; + end = smartlist_len(cons1); + ++ptr; + } else if (get_linenum(&ptr, &end) < 0) { log_warn(LD_CONSDIFF, "Could not apply consensus diff because " "an ed command was missing a range end line number."); goto error_cleanup; @@ -850,6 +937,20 @@ apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, goto error_cleanup; } + /** $ is not allowed with non-d actions. */ + if (end_was_eof && action != 'd') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it wanted to use $ with a command other than delete"); + goto error_cleanup; + } + + /* 'a' commands are not allowed to have ranges. */ + if (had_range && action == 'a') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it wanted to add lines after a range."); + goto error_cleanup; + } + /* Add unchanged lines. */ for (; j && j > end; --j) { cdline_t *cons_line = smartlist_get(cons1, j-1); @@ -1238,7 +1339,7 @@ consensus_diff_generate(const char *cons1, int r1, r2; char *result = NULL; - r1 = consensus_compute_digest(cons1, &d1); + r1 = consensus_compute_digest_as_signed(cons1, &d1); r2 = consensus_compute_digest(cons2, &d2); if (BUG(r1 < 0 || r2 < 0)) return NULL; // LCOV_EXCL_LINE @@ -1279,7 +1380,7 @@ consensus_diff_apply(const char *consensus, char *result = NULL; memarea_t *area = memarea_new(); - r1 = consensus_compute_digest(consensus, &d1); + r1 = consensus_compute_digest_as_signed(consensus, &d1); if (BUG(r1 < 0)) return NULL; // LCOV_EXCL_LINE @@ -1300,3 +1401,12 @@ consensus_diff_apply(const char *consensus, return result; } +/** Return true iff, based on its header, <b>document</b> is likely + * to be a consensus diff. */ +int +looks_like_a_consensus_diff(const char *document, size_t len) +{ + return (len >= strlen(ns_diff_version) && + fast_memeq(document, ns_diff_version, strlen(ns_diff_version))); +} + diff --git a/src/or/consdiff.h b/src/or/consdiff.h index e9d175136e..d05df74b75 100644 --- a/src/or/consdiff.h +++ b/src/or/consdiff.h @@ -12,6 +12,8 @@ char *consensus_diff_generate(const char *cons1, char *consensus_diff_apply(const char *consensus, const char *diff); +int looks_like_a_consensus_diff(const char *document, size_t len); + #ifdef CONSDIFF_PRIVATE struct memarea_t; @@ -85,6 +87,9 @@ MOCK_DECL(STATIC int, consensus_compute_digest,(const char *cons, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, + consensus_compute_digest_as_signed,(const char *cons, + consensus_digest_t *digest_out)); +MOCK_DECL(STATIC int, consensus_digest_eq,(const uint8_t *d1, const uint8_t *d2)); #endif diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c new file mode 100644 index 0000000000..d478101c6b --- /dev/null +++ b/src/or/consdiffmgr.c @@ -0,0 +1,1410 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file consdiffmsr.c + * + * \brief consensus diff manager functions + * + * This module is run by directory authorities and caches in order + * to remember a number of past consensus documents, and to generate + * and serve the diffs from those documents to the latest consensus. + */ + +#define CONSDIFFMGR_PRIVATE + +#include "or.h" +#include "conscache.h" +#include "consdiff.h" +#include "consdiffmgr.h" +#include "cpuworker.h" +#include "networkstatus.h" +#include "routerparse.h" +#include "workqueue.h" + +/** + * Labels to apply to items in the conscache object. + * + * @{ + */ +/* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */ +#define LABEL_DOCTYPE "document-type" +/* The valid-after time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_VALID_AFTER "consensus-valid-after" +/* A hex encoded SHA3 digest of the object, as compressed (if any) */ +#define LABEL_SHA3_DIGEST "sha3-digest" +/* A hex encoded SHA3 digest of the object before compression. */ +#define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed" +/* A hex encoded SHA3 digest-as-signed of a consensus */ +#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed" +/* The flavor of the consensus or consensuses diff */ +#define LABEL_FLAVOR "consensus-flavor" +/* Diff only: the SHA3 digest-as-signed of the source consensus. */ +#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest" +/* Diff only: the SHA3 digest-in-full of the target consensus. */ +#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest" +/* Diff only: the valid-after date of the source consensus. */ +#define LABEL_FROM_VALID_AFTER "from-valid-after" +/* What kind of compression was used? */ +#define LABEL_COMPRESSION_TYPE "compression" +/** @} */ + +#define DOCTYPE_CONSENSUS "consensus" +#define DOCTYPE_CONSENSUS_DIFF "consensus-diff" + +/** + * Underlying directory that stores consensuses and consensus diffs. Don't + * use this directly: use cdm_cache_get() instead. + */ +static consensus_cache_t *cons_diff_cache = NULL; +/** + * If true, we have learned at least one new consensus since the + * consensus cache was last up-to-date. + */ +static int cdm_cache_dirty = 0; +/** + * If true, we have scanned the cache to update our hashtable of diffs. + */ +static int cdm_cache_loaded = 0; + +/** + * Possible status values for cdm_diff_t.cdm_diff_status + **/ +typedef enum cdm_diff_status_t { + CDM_DIFF_PRESENT=1, + CDM_DIFF_IN_PROGRESS=2, + CDM_DIFF_ERROR=3, +} cdm_diff_status_t; + +/** Which methods do we use for precompressing diffs? */ +static const compress_method_t compress_diffs_with[] = { + NO_METHOD, + GZIP_METHOD, +#ifdef HAVE_LZMA + LZMA_METHOD, +#endif +#ifdef HAVE_ZSTD + ZSTD_METHOD, +#endif +}; + +/** How many different methods will we try to use for diff compression? */ +STATIC unsigned +n_diff_compression_methods(void) +{ + return ARRAY_LENGTH(compress_diffs_with); +} + +/** Hashtable node used to remember the current status of the diff + * from a given sha3 digest to the current consensus. */ +typedef struct cdm_diff_t { + HT_ENTRY(cdm_diff_t) node; + + /** Consensus flavor for this diff (part of ht key) */ + consensus_flavor_t flavor; + /** SHA3-256 digest of the consensus that this diff is _from_. (part of the + * ht key) */ + uint8_t from_sha3[DIGEST256_LEN]; + /** Method by which the diff is compressed. (part of the ht key */ + compress_method_t compress_method; + + /** One of the CDM_DIFF_* values, depending on whether this diff + * is available, in progress, or impossible to compute. */ + cdm_diff_status_t cdm_diff_status; + /** SHA3-256 digest of the consensus that this diff is _to. */ + uint8_t target_sha3[DIGEST256_LEN]; + + /** Handle to the cache entry for this diff, if any. We use a handle here + * to avoid thinking too hard about cache entry lifetime issues. */ + consensus_cache_entry_handle_t *entry; +} cdm_diff_t; + +/** Hashtable mapping flavor and source consensus digest to status. */ +static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER(); + +/** + * Configuration for this module + */ +static consdiff_cfg_t consdiff_cfg = { + /* .cache_max_age_hours = */ 24 * 90, + // XXXX I'd like to make this number bigger, but it interferes with the + // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096) + // XXXX rules. + /* .cache_max_num = */ 128 +}; + +static int consdiffmgr_ensure_space_for_files(int n); +static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, + consensus_cache_entry_t *diff_to); +static void consdiffmgr_set_cache_flags(void); + +/* Just gzip consensuses for now. */ +#define COMPRESS_CONSENSUS_WITH GZIP_METHOD + +/* ===== + * Hashtable setup + * ===== */ + +/** Helper: hash the key of a cdm_diff_t. */ +static unsigned +cdm_diff_hash(const cdm_diff_t *diff) +{ + uint8_t tmp[DIGEST256_LEN + 2]; + memcpy(tmp, diff->from_sha3, DIGEST256_LEN); + tmp[DIGEST256_LEN] = (uint8_t) diff->flavor; + tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method; + return (unsigned) siphash24g(tmp, sizeof(tmp)); +} +/** Helper: compare two cdm_diff_t objects for key equality */ +static int +cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2) +{ + return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) && + diff1->flavor == diff2->flavor && + diff1->compress_method == diff2->compress_method; +} + +HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq) +HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq, + 0.6, tor_reallocarray, tor_free_) + +/** Release all storage held in <b>diff</b>. */ +static void +cdm_diff_free(cdm_diff_t *diff) +{ + if (!diff) + return; + consensus_cache_entry_handle_free(diff->entry); + tor_free(diff); +} + +/** Create and return a new cdm_diff_t with the given values. Does not + * add it to the hashtable. */ +static cdm_diff_t * +cdm_diff_new(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *target_sha3, + compress_method_t method) +{ + cdm_diff_t *ent; + ent = tor_malloc_zero(sizeof(cdm_diff_t)); + ent->flavor = flav; + memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN); + memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN); + ent->compress_method = method; + return ent; +} + +/** + * Examine the diff hashtable to see whether we know anything about computing + * a diff of type <b>flav</b> between consensuses with the two provided + * SHA3-256 digests. If a computation is in progress, or if the computation + * has already been tried and failed, return 1. Otherwise, note the + * computation as "in progress" so that we don't reattempt it later, and + * return 0. + */ +static int +cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *target_sha3) +{ + struct cdm_diff_t search, *ent; + unsigned u; + int result = 0; + for (u = 0; u < n_diff_compression_methods(); ++u) { + compress_method_t method = compress_diffs_with[u]; + memset(&search, 0, sizeof(cdm_diff_t)); + search.flavor = flav; + search.compress_method = method; + memcpy(search.from_sha3, from_sha3, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + if (ent) { + tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT); + result = 1; + continue; + } + ent = cdm_diff_new(flav, from_sha3, target_sha3, method); + ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS; + HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent); + } + return result; +} + +/** + * Update the status of the diff of type <b>flav</b> between consensuses with + * the two provided SHA3-256 digests, so that its status becomes + * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b> + * is NULL, then the old handle (if any) is freed, and replaced with NULL. + */ +static void +cdm_diff_ht_set_status(consensus_flavor_t flav, + const uint8_t *from_sha3, + const uint8_t *to_sha3, + compress_method_t method, + int status, + consensus_cache_entry_handle_t *handle) +{ + struct cdm_diff_t search, *ent; + memset(&search, 0, sizeof(cdm_diff_t)); + search.flavor = flav; + search.compress_method = method, + memcpy(search.from_sha3, from_sha3, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + if (!ent) { + ent = cdm_diff_new(flav, from_sha3, to_sha3, method); + ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS; + HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent); + } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) { + // This can happen under certain really pathological conditions + // if we decide we don't care about a diff before it is actually + // done computing. + return; + } + + tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS); + + ent->cdm_diff_status = status; + consensus_cache_entry_handle_free(ent->entry); + ent->entry = handle; +} + +/** + * Helper: Remove from the hash table every present (actually computed) diff + * of type <b>flav</b> whose target digest does not match + * <b>unless_target_sha3_matches</b>. + * + * This function is used for the hash table to throw away references to diffs + * that do not lead to the most given consensus of a given flavor. + */ +static void +cdm_diff_ht_purge(consensus_flavor_t flav, + const uint8_t *unless_target_sha3_matches) +{ + cdm_diff_t **diff, **next; + for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) { + cdm_diff_t *this = *diff; + + if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT && + flav == (*diff)->flavor) { + + if (consensus_cache_entry_handle_get((*diff)->entry) == NULL) { + /* the underlying entry has gone away; drop this. */ + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + continue; + } + + if (unless_target_sha3_matches && + fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3, + DIGEST256_LEN)) { + /* target hash doesn't match; drop this. */ + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + continue; + } + } + next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff); + } +} + +/** + * Helper: initialize <b>cons_diff_cache</b>. + */ +static void +cdm_cache_init(void) +{ + unsigned n_entries = consdiff_cfg.cache_max_num * 2; + + tor_assert(cons_diff_cache == NULL); + cons_diff_cache = consensus_cache_open("diff-cache", n_entries); + if (cons_diff_cache == NULL) { + // LCOV_EXCL_START + log_err(LD_FS, "Error: Couldn't open storage for consensus diffs."); + tor_assert_unreached(); + // LCOV_EXCL_STOP + } else { + consdiffmgr_set_cache_flags(); + } + cdm_cache_dirty = 1; + cdm_cache_loaded = 0; +} + +/** + * Helper: return the consensus_cache_t * that backs this manager, + * initializing it if needed. + */ +STATIC consensus_cache_t * +cdm_cache_get(void) +{ + if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) { + cdm_cache_init(); + } + return cons_diff_cache; +} + +/** + * Helper: given a list of labels, prepend the hex-encoded SHA3 digest + * of the <b>bodylen</b>-byte object at <b>body</b> to those labels, + * with <b>label</b> as its label. + */ +static void +cdm_labels_prepend_sha3(config_line_t **labels, + const char *label, + const uint8_t *body, + size_t bodylen) +{ + uint8_t sha3_digest[DIGEST256_LEN]; + char hexdigest[HEX_DIGEST256_LEN+1]; + crypto_digest256((char *)sha3_digest, + (const char *)body, bodylen, DIGEST_SHA3_256); + base16_encode(hexdigest, sizeof(hexdigest), + (const char *)sha3_digest, sizeof(sha3_digest)); + + config_line_prepend(labels, label, hexdigest); +} + +/** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the + * given label, set <b>digest_out</b> to that value (decoded), and return 0. + * + * Return -1 if there is no such label, and -2 if it is badly formatted. */ +STATIC int +cdm_entry_get_sha3_value(uint8_t *digest_out, + consensus_cache_entry_t *ent, + const char *label) +{ + if (ent == NULL) + return -1; + + const char *hex = consensus_cache_entry_get_value(ent, label); + if (hex == NULL) + return -1; + + int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex)); + if (n != DIGEST256_LEN) + return -2; + else + return 0; +} + +/** + * Helper: look for a consensus with the given <b>flavor</b> and + * <b>valid_after</b> time in the cache. Return that consensus if it's + * present, or NULL if it's missing. + */ +STATIC consensus_cache_entry_t * +cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after) +{ + char formatted_time[ISO_TIME_LEN+1]; + format_iso_time_nospace(formatted_time, valid_after); + const char *flavname = networkstatus_get_flavor_name(flavor); + + /* We'll filter by valid-after time first, since that should + * match the fewest documents. */ + /* We could add an extra hashtable here, but since we only do this scan + * when adding a new consensus, it probably doesn't matter much. */ + smartlist_t *matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_VALID_AFTER, formatted_time); + consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + consensus_cache_entry_t *result = NULL; + if (smartlist_len(matches) > 1) { + log_warn(LD_BUG, "How odd; there appear to be two matching consensuses " + "with flavor %s published at %s.", + flavname, formatted_time); + } + if (smartlist_len(matches)) { + result = smartlist_get(matches, 0); + } + smartlist_free(matches); + + return result; +} + +/** + * Given a string containing a networkstatus consensus, and the results of + * having parsed that consensus, add that consensus to the cache if it is not + * already present and not too old. Create new consensus diffs from or to + * that consensus as appropriate. + * + * Return 0 on success and -1 on failure. + */ +int +consdiffmgr_add_consensus(const char *consensus, + const networkstatus_t *as_parsed) +{ + if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) + return -1; // LCOV_EXCL_LINE + if (BUG(as_parsed->type != NS_TYPE_CONSENSUS)) + return -1; // LCOV_EXCL_LINE + + const consensus_flavor_t flavor = as_parsed->flavor; + const time_t valid_after = as_parsed->valid_after; + + if (valid_after < approx_time() - 3600 * consdiff_cfg.cache_max_age_hours) { + log_info(LD_DIRSERV, "We don't care about this consensus document; it's " + "too old."); + return -1; + } + + /* Do we already have this one? */ + consensus_cache_entry_t *entry = + cdm_cache_lookup_consensus(flavor, valid_after); + if (entry) { + log_info(LD_DIRSERV, "We already have a copy of that consensus"); + return -1; + } + + /* We don't have it. Add it to the cache. */ + consdiffmgr_ensure_space_for_files(1); + + { + size_t bodylen = strlen(consensus); + config_line_t *labels = NULL; + char formatted_time[ISO_TIME_LEN+1]; + format_iso_time_nospace(formatted_time, valid_after); + const char *flavname = networkstatus_get_flavor_name(flavor); + + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, + (const uint8_t *)consensus, bodylen); + { + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(consensus, + &start, &end) < 0) { + start = consensus; + end = consensus+bodylen; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, + (const uint8_t *)start, + end - start); + } + + char *body_compressed = NULL; + size_t size_compressed = 0; + if (tor_compress(&body_compressed, &size_compressed, + consensus, bodylen, COMPRESS_CONSENSUS_WITH) < 0) { + config_free_lines(labels); + return -1; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST, + (const uint8_t *)body_compressed, size_compressed); + config_line_prepend(&labels, LABEL_COMPRESSION_TYPE, + compression_method_get_name(COMPRESS_CONSENSUS_WITH)); + config_line_prepend(&labels, LABEL_FLAVOR, flavname); + config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); + config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + entry = consensus_cache_add(cdm_cache_get(), + labels, + (const uint8_t *)body_compressed, + size_compressed); + tor_free(body_compressed); + config_free_lines(labels); + } + + if (entry) { + consensus_cache_entry_mark_for_aggressive_release(entry); + consensus_cache_entry_decref(entry); + } + + cdm_cache_dirty = 1; + return entry ? 0 : -1; +} + +/** + * Helper: used to sort two smartlists of consensus_cache_entry_t by their + * LABEL_VALID_AFTER labels. + */ +static int +compare_by_valid_after_(const void **a, const void **b) +{ + const consensus_cache_entry_t *e1 = *a; + const consensus_cache_entry_t *e2 = *b; + /* We're in luck here: sorting UTC iso-encoded values lexically will work + * fine (until 9999). */ + return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER), + consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER)); +} + +/** + * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent + * entry. + */ +static consensus_cache_entry_t * +sort_and_find_most_recent(smartlist_t *lst) +{ + smartlist_sort(lst, compare_by_valid_after_); + if (smartlist_len(lst)) { + return smartlist_get(lst, smartlist_len(lst) - 1); + } else { + return NULL; + } +} + +/** + * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>, + * from the source consensus with the specified digest (which must be SHA3). + * + * If the diff is present, store it into *<b>entry_out</b> and return + * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or + * CONSDIFF_IN_PROGRESS. + */ +consdiff_status_t +consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + int digest_type, + const uint8_t *digest, + size_t digestlen, + compress_method_t method) +{ + if (BUG(digest_type != DIGEST_SHA3_256) || + BUG(digestlen != DIGEST256_LEN)) { + return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE + } + + // Try to look up the entry in the hashtable. + cdm_diff_t search, *ent; + memset(&search, 0, sizeof(search)); + search.flavor = flavor; + search.compress_method = method; + memcpy(search.from_sha3, digest, DIGEST256_LEN); + ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search); + + if (ent == NULL || + ent->cdm_diff_status == CDM_DIFF_ERROR) { + return CONSDIFF_NOT_FOUND; + } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) { + return CONSDIFF_IN_PROGRESS; + } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) { + return CONSDIFF_IN_PROGRESS; + } + + *entry_out = consensus_cache_entry_handle_get(ent->entry); + return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND; + +#if 0 + // XXXX Remove this. I'm keeping it around for now in case we need to + // XXXX debug issues in the hashtable. + char hex[HEX_DIGEST256_LEN+1]; + base16_encode(hex, sizeof(hex), (const char *)digest, digestlen); + const char *flavname = networkstatus_get_flavor_name(flavor); + + smartlist_t *matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FROM_SHA3_DIGEST, hex); + consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + + *entry_out = sort_and_find_most_recent(matches); + consdiff_status_t result = + (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND; + smartlist_free(matches); + + return result; +#endif +} + +/** + * Perform periodic cleanup tasks on the consensus diff cache. Return + * the number of objects marked for deletion. + */ +int +consdiffmgr_cleanup(void) +{ + smartlist_t *objects = smartlist_new(); + smartlist_t *consensuses = smartlist_new(); + smartlist_t *diffs = smartlist_new(); + int n_to_delete = 0; + + log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove"); + + // 1. Delete any consensus or diff or anything whose valid_after is too old. + const time_t valid_after_cutoff = + approx_time() - 3600 * consdiff_cfg.cache_max_age_hours; + + consensus_cache_find_all(objects, cdm_cache_get(), + NULL, NULL); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + const char *lv_valid_after = + consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER); + if (! lv_valid_after) { + log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label", + LABEL_VALID_AFTER); + continue; + } + time_t valid_after = 0; + if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) { + log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was " + "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after)); + continue; + } + if (valid_after < valid_after_cutoff) { + log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was " + "too old", LABEL_VALID_AFTER, lv_valid_after); + consensus_cache_entry_mark_for_removal(ent); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(ent); + + // 2. Delete all diffs that lead to a consensus whose valid-after is not the + // latest. + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + /* Determine the most recent consensus of this flavor */ + consensus_cache_find_all(consensuses, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname); + consensus_cache_entry_t *most_recent = + sort_and_find_most_recent(consensuses); + if (most_recent == NULL) + continue; + const char *most_recent_sha3 = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(most_recent_sha3 == NULL)) + continue; // LCOV_EXCL_LINE + + /* consider all such-flavored diffs, and look to see if they match. */ + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *this_diff_target_sha3 = + consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST); + if (!this_diff_target_sha3) + continue; + if (strcmp(this_diff_target_sha3, most_recent_sha3)) { + consensus_cache_entry_mark_for_removal(diff); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(diff); + smartlist_clear(consensuses); + smartlist_clear(diffs); + } + + smartlist_free(objects); + smartlist_free(consensuses); + smartlist_free(diffs); + + // Actually remove files, if they're not used. + consensus_cache_delete_pending(cdm_cache_get(), 0); + return n_to_delete; +} + +/** + * Initialize the consensus diff manager and its cache, and configure + * its parameters based on the latest torrc and networkstatus parameters. + */ +void +consdiffmgr_configure(const consdiff_cfg_t *cfg) +{ + if (cfg) + memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg)); + + (void) cdm_cache_get(); +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that the consensus diff manager will need. + */ +int +consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg) +{ + return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg); +} + +/** + * Scan the consensus diff manager's cache for any grossly malformed entries, + * and mark them as deletable. Return 0 if no problems were found; 1 + * if problems were found and fixed. + */ +int +consdiffmgr_validate(void) +{ + /* Right now, we only check for entries that have bad sha3 values */ + int problems = 0; + + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cdm_cache_get(), + NULL, NULL); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) { + uint8_t sha3_expected[DIGEST256_LEN]; + uint8_t sha3_received[DIGEST256_LEN]; + int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST); + if (r == -1) { + /* digest isn't there; that's allowed */ + continue; + } else if (r == -2) { + /* digest is malformed; that's not allowed */ + problems = 1; + consensus_cache_entry_mark_for_removal(obj); + continue; + } + const uint8_t *body; + size_t bodylen; + consensus_cache_entry_incref(obj); + r = consensus_cache_entry_get_body(obj, &body, &bodylen); + if (r == 0) { + crypto_digest256((char *)sha3_received, (const char *)body, bodylen, + DIGEST_SHA3_256); + } + consensus_cache_entry_decref(obj); + if (r < 0) + continue; + + // Deconfuse coverity about the possibility of sha3_received being + // uninitialized + tor_assert(r <= 0); + + if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) { + problems = 1; + consensus_cache_entry_mark_for_removal(obj); + continue; + } + + } SMARTLIST_FOREACH_END(obj); + smartlist_free(objects); + return problems; +} + +/** + * Helper: build new diffs of <b>flavor</b> as needed + */ +static void +consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) +{ + smartlist_t *matches = NULL; + smartlist_t *diffs = NULL; + smartlist_t *compute_diffs_from = NULL; + strmap_t *have_diff_from = NULL; + + // look for the most recent consensus, and for all previous in-range + // consensuses. Do they all have diffs to it? + const char *flavname = networkstatus_get_flavor_name(flavor); + + // 1. find the most recent consensus, and the ones that we might want + // to diff to it. + matches = smartlist_new(); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); + if (!most_recent) { + log_info(LD_DIRSERV, "No 'most recent' %s consensus found; " + "not making diffs", flavname); + goto done; + } + tor_assert(smartlist_len(matches)); + smartlist_del(matches, smartlist_len(matches) - 1); + + const char *most_recent_valid_after = + consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER); + if (BUG(most_recent_valid_after == NULL)) + goto done; //LCOV_EXCL_LINE + uint8_t most_recent_sha3[DIGEST256_LEN]; + if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0)) + goto done; //LCOV_EXCL_LINE + + // 2. Find all the relevant diffs _to_ this consensus. These are ones + // that we don't need to compute. + diffs = smartlist_new(); + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_VALID_AFTER, most_recent_valid_after); + consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname); + have_diff_from = strmap_new(); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *va = consensus_cache_entry_get_value(diff, + LABEL_FROM_VALID_AFTER); + if (BUG(va == NULL)) + continue; // LCOV_EXCL_LINE + strmap_set(have_diff_from, va, diff); + } SMARTLIST_FOREACH_END(diff); + + // 3. See which consensuses in 'matches' don't have diffs yet. + smartlist_reverse(matches); // from newest to oldest. + compute_diffs_from = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) { + const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER); + if (BUG(va == NULL)) + continue; // LCOV_EXCL_LINE + if (strmap_get(have_diff_from, va) != NULL) + continue; /* we already have this one. */ + smartlist_add(compute_diffs_from, ent); + } SMARTLIST_FOREACH_END(ent); + + log_info(LD_DIRSERV, + "The most recent %s consensus is valid-after %s. We have diffs to " + "this consensus for %d/%d older %s consensuses. Generating diffs " + "for the other %d.", + flavname, + most_recent_valid_after, + smartlist_len(matches) - smartlist_len(compute_diffs_from), + smartlist_len(matches), + flavname, + smartlist_len(compute_diffs_from)); + + // 4. Update the hashtable; remove entries in this flavor to other + // target consensuses. + cdm_diff_ht_purge(flavor, most_recent_sha3); + + // 5. Actually launch the requests. + SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) { + if (BUG(c == most_recent)) + continue; // LCOV_EXCL_LINE + + uint8_t this_sha3[DIGEST256_LEN]; + if (cdm_entry_get_sha3_value(this_sha3, c, + LABEL_SHA3_DIGEST_AS_SIGNED)<0) { + // Not actually a bug, since we might be running with a directory + // with stale files from before the #22143 fixes. + continue; + } + if (cdm_diff_ht_check_and_note_pending(flavor, + this_sha3, most_recent_sha3)) { + // This is already pending, or we encountered an error. + continue; + } + consensus_diff_queue_diff_work(c, most_recent); + } SMARTLIST_FOREACH_END(c); + + done: + smartlist_free(matches); + smartlist_free(diffs); + smartlist_free(compute_diffs_from); + strmap_free(have_diff_from, NULL); +} + +/** + * Scan the cache for diffs, and add them to the hashtable. + */ +static void +consdiffmgr_diffs_load(void) +{ + smartlist_t *diffs = smartlist_new(); + consensus_cache_find_all(diffs, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF); + SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) { + const char *lv_flavor = + consensus_cache_entry_get_value(diff, LABEL_FLAVOR); + if (!lv_flavor) + continue; + int flavor = networkstatus_parse_flavor_name(lv_flavor); + if (flavor < 0) + continue; + const char *lv_compression = + consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE); + compress_method_t method = NO_METHOD; + if (lv_compression) { + method = compression_method_get_by_name(lv_compression); + if (method == UNKNOWN_METHOD) { + continue; + } + } + + uint8_t from_sha3[DIGEST256_LEN]; + uint8_t to_sha3[DIGEST256_LEN]; + if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0) + continue; + if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0) + continue; + + cdm_diff_ht_set_status(flavor, from_sha3, to_sha3, + method, + CDM_DIFF_PRESENT, + consensus_cache_entry_handle_new(diff)); + } SMARTLIST_FOREACH_END(diff); + smartlist_free(diffs); +} + +/** + * Build new diffs as needed. + */ +void +consdiffmgr_rescan(void) +{ + if (cdm_cache_dirty == 0) + return; + + // Clean up here to make room for new diffs, and to ensure that older + // consensuses do not have any entries. + consdiffmgr_cleanup(); + + if (cdm_cache_loaded == 0) { + consdiffmgr_diffs_load(); + cdm_cache_loaded = 1; + } + + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + consdiffmgr_rescan_flavor_((consensus_flavor_t) flav); + } + + cdm_cache_dirty = 0; +} + +/** + * Helper: compare two files by their from-valid-after and valid-after labels, + * trying to sort in ascending order by from-valid-after (when present) and + * valid-after (when not). Place everything that has neither label first in + * the list. + */ +static int +compare_by_staleness_(const void **a, const void **b) +{ + const consensus_cache_entry_t *e1 = *a; + const consensus_cache_entry_t *e2 = *b; + const char *va1, *fva1, *va2, *fva2; + va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER); + va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER); + fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER); + fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER); + + if (fva1) + va1 = fva1; + if (fva2) + va2 = fva2; + + /* See note about iso-encoded values in compare_by_valid_after_. Also note + * that missing dates will get placed first. */ + return strcmp_opt(va1, va2); +} + +/** If there are not enough unused filenames to store <b>n</b> files, then + * delete old consensuses until there are. (We have to keep track of the + * number of filenames because of the way that the seccomp2 cache works.) + * + * Return 0 on success, -1 on failure. + **/ +static int +consdiffmgr_ensure_space_for_files(int n) +{ + consensus_cache_t *cache = cdm_cache_get(); + if (consensus_cache_get_n_filenames_available(cache) >= n) { + // there are already enough unused filenames. + return 0; + } + // Try a cheap deletion of stuff that's waiting to get deleted. + consensus_cache_delete_pending(cache, 0); + if (consensus_cache_get_n_filenames_available(cache) >= n) { + // okay, _that_ made enough filenames available. + return 0; + } + // Let's get more assertive: clean out unused stuff, and force-remove + // the files. + consdiffmgr_cleanup(); + consensus_cache_delete_pending(cache, 1); + const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache); + if (n_to_remove <= 0) { + // okay, finally! + return 0; + } + + // At this point, we're going to have to throw out objects that will be + // missed. Too bad! + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cache, NULL, NULL); + smartlist_sort(objects, compare_by_staleness_); + int n_marked = 0; + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + consensus_cache_entry_mark_for_removal(ent); + if (++n_marked >= n_to_remove) + break; + } SMARTLIST_FOREACH_END(ent); + + consensus_cache_delete_pending(cache, 1); + if (BUG(n_marked < n_to_remove)) + return -1; + else + return 0; +} + +/** + * Set consensus cache flags on the objects in this consdiffmgr. + */ +static void +consdiffmgr_set_cache_flags(void) +{ + /* Right now, we just mark the consensus objects for aggressive release, + * so that they get mmapped for as little time as possible. */ + smartlist_t *objects = smartlist_new(); + consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE, + DOCTYPE_CONSENSUS); + SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) { + consensus_cache_entry_mark_for_aggressive_release(ent); + } SMARTLIST_FOREACH_END(ent); + smartlist_free(objects); +} + +/** + * Called before shutdown: drop all storage held by the consdiffmgr.c module. + */ +void +consdiffmgr_free_all(void) +{ + cdm_diff_t **diff, **next; + for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) { + cdm_diff_t *this = *diff; + next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); + cdm_diff_free(this); + } + consensus_cache_free(cons_diff_cache); + cons_diff_cache = NULL; +} + +/* ===== + Thread workers + =====*/ + +typedef struct compressed_result_t { + config_line_t *labels; + /** + * Output: Body of the diff, as compressed. + */ + uint8_t *body; + /** + * Output: length of body_out + */ + size_t bodylen; +} compressed_result_t; + +/** + * An object passed to a worker thread that will try to produce a consensus + * diff. + */ +typedef struct consensus_diff_worker_job_t { + /** + * Input: The consensus to compute the diff from. Holds a reference to the + * cache entry, which must not be released until the job is passed back to + * the main thread. The body must be mapped into memory in the main thread. + */ + consensus_cache_entry_t *diff_from; + /** + * Input: The consensus to compute the diff to. Holds a reference to the + * cache entry, which must not be released until the job is passed back to + * the main thread. The body must be mapped into memory in the main thread. + */ + consensus_cache_entry_t *diff_to; + + /** Output: labels and bodies */ + compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)]; +} consensus_diff_worker_job_t; + +/** Given a consensus_cache_entry_t, check whether it has a label claiming + * that it was compressed. If so, uncompress its contents into <b>out</b> and + * set <b>outlen</b> to hold their size. If not, just copy the body into + * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success, + * -1 on failure. + * + * In all cases, the output is nul-terminated. */ +STATIC int +uncompress_or_copy(char **out, size_t *outlen, + consensus_cache_entry_t *ent) +{ + const uint8_t *body; + size_t bodylen; + + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) + return -1; + + const char *lv_compression = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + compress_method_t method = NO_METHOD; + + if (lv_compression) + method = compression_method_get_by_name(lv_compression); + + return tor_uncompress(out, outlen, (const char *)body, bodylen, + method, 1, LOG_WARN); +} + +/** + * Worker function. This function runs inside a worker thread and receives + * a consensus_diff_worker_job_t as its input. + */ +static workqueue_reply_t +consensus_diff_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + consensus_diff_worker_job_t *job = work_; + const uint8_t *diff_from, *diff_to; + size_t len_from, len_to; + int r; + /* We need to have the body already mapped into RAM here. + */ + r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from); + if (BUG(r < 0)) + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to); + if (BUG(r < 0)) + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + + const char *lv_to_valid_after = + consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER); + const char *lv_from_valid_after = + consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER); + const char *lv_from_digest = + consensus_cache_entry_get_value(job->diff_from, + LABEL_SHA3_DIGEST_AS_SIGNED); + const char *lv_from_flavor = + consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR); + const char *lv_to_flavor = + consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR); + const char *lv_to_digest = + consensus_cache_entry_get_value(job->diff_to, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + + if (! lv_from_digest) { + /* This isn't a bug right now, since it can happen if you're migrating + * from an older version of master to a newer one. The older ones didn't + * annotate their stored consensus objects with sha3-digest-as-signed. + */ + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + + /* All these values are mandatory on the input */ + if (BUG(!lv_to_valid_after) || + BUG(!lv_from_valid_after) || + BUG(!lv_from_flavor) || + BUG(!lv_to_flavor)) { + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + /* The flavors need to match */ + if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) { + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + + char *consensus_diff; + { + char *diff_from_nt = NULL, *diff_to_nt = NULL; + size_t diff_from_nt_len, diff_to_nt_len; + + if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, + job->diff_from) < 0) { + return WQ_RPL_REPLY; + } + if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, + job->diff_to) < 0) { + tor_free(diff_from_nt); + return WQ_RPL_REPLY; + } + tor_assert(diff_from_nt); + tor_assert(diff_to_nt); + + // XXXX ugh; this is going to calculate the SHA3 of both its + // XXXX inputs again, even though we already have that. Maybe it's time + // XXXX to change the API here? + consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); + tor_free(diff_from_nt); + tor_free(diff_to_nt); + } + if (!consensus_diff) { + /* Couldn't generate consensus; we'll leave the reply blank. */ + return WQ_RPL_REPLY; + } + + /* Compress the results and send the reply */ + tor_assert(compress_diffs_with[0] == NO_METHOD); + size_t difflen = strlen(consensus_diff); + job->out[0].body = (uint8_t *) consensus_diff; + job->out[0].bodylen = difflen; + + config_line_t *common_labels = NULL; + cdm_labels_prepend_sha3(&common_labels, + LABEL_SHA3_DIGEST_UNCOMPRESSED, + job->out[0].body, + job->out[0].bodylen); + config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER, + lv_from_valid_after); + config_line_prepend(&common_labels, LABEL_VALID_AFTER, + lv_to_valid_after); + config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor); + config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST, + lv_from_digest); + config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST, + lv_to_digest); + config_line_prepend(&common_labels, LABEL_DOCTYPE, + DOCTYPE_CONSENSUS_DIFF); + + job->out[0].labels = config_lines_dup(common_labels); + cdm_labels_prepend_sha3(&job->out[0].labels, + LABEL_SHA3_DIGEST, + job->out[0].body, + job->out[0].bodylen); + + unsigned u; + for (u = 1; u < n_diff_compression_methods(); ++u) { + compress_method_t method = compress_diffs_with[u]; + const char *methodname = compression_method_get_name(method); + char *result; + size_t sz; + if (0 == tor_compress(&result, &sz, consensus_diff, difflen, method)) { + job->out[u].body = (uint8_t*)result; + job->out[u].bodylen = sz; + job->out[u].labels = config_lines_dup(common_labels); + cdm_labels_prepend_sha3(&job->out[u].labels, LABEL_SHA3_DIGEST, + job->out[u].body, + job->out[u].bodylen); + config_line_prepend(&job->out[u].labels, + LABEL_COMPRESSION_TYPE, + methodname); + } + } + + config_free_lines(common_labels); + return WQ_RPL_REPLY; +} + +/** + * Helper: release all storage held in <b>job</b>. + */ +static void +consensus_diff_worker_job_free(consensus_diff_worker_job_t *job) +{ + if (!job) + return; + unsigned u; + for (u = 0; u < n_diff_compression_methods(); ++u) { + config_free_lines(job->out[u].labels); + tor_free(job->out[u].body); + } + consensus_cache_entry_decref(job->diff_from); + consensus_cache_entry_decref(job->diff_to); + tor_free(job); +} + +/** + * Worker function: This function runs in the main thread, and receives + * a consensus_diff_worker_job_t that the worker thread has already + * processed. + */ +static void +consensus_diff_worker_replyfn(void *work_) +{ + tor_assert(in_main_thread()); + tor_assert(work_); + + consensus_diff_worker_job_t *job = work_; + + const char *lv_from_digest = + consensus_cache_entry_get_value(job->diff_from, + LABEL_SHA3_DIGEST_AS_SIGNED); + const char *lv_to_digest = + consensus_cache_entry_get_value(job->diff_to, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + const char *lv_flavor = + consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR); + if (BUG(lv_from_digest == NULL)) + lv_from_digest = "???"; // LCOV_EXCL_LINE + if (BUG(lv_to_digest == NULL)) + lv_to_digest = "???"; // LCOV_EXCL_LINE + + uint8_t from_sha3[DIGEST256_LEN]; + uint8_t to_sha3[DIGEST256_LEN]; + int flav = -1; + int cache = 1; + if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from, + LABEL_SHA3_DIGEST_AS_SIGNED) < 0)) + cache = 0; + if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to, + LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0)) + cache = 0; + if (BUG(lv_flavor == NULL)) { + cache = 0; + } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) { + cache = 0; + } + + int status = CDM_DIFF_ERROR; + consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)]; + memset(handles, 0, sizeof(handles)); + + consdiffmgr_ensure_space_for_files(n_diff_compression_methods()); + + unsigned u; + for (u = 0; u < n_diff_compression_methods(); ++u) { + compress_method_t method = compress_diffs_with[u]; + uint8_t *body_out = job->out[u].body; + size_t bodylen_out = job->out[u].bodylen; + config_line_t *labels = job->out[u].labels; + const char *methodname = compression_method_get_name(method); + if (body_out && bodylen_out && labels) { + /* Success! Store the results */ + log_info(LD_DIRSERV, "Adding consensus diff from %s to %s, " + "compressed with %s", + lv_from_digest, lv_to_digest, methodname); + + consensus_cache_entry_t *ent = + consensus_cache_add(cdm_cache_get(), + labels, + body_out, + bodylen_out); + + status = CDM_DIFF_PRESENT; + handles[u] = consensus_cache_entry_handle_new(ent); + consensus_cache_entry_decref(ent); + } + } + if (status != CDM_DIFF_PRESENT) { + /* Failure! Nothing to do but complain */ + log_warn(LD_DIRSERV, + "Worker was unable to compute consensus diff " + "from %s to %s", lv_from_digest, lv_to_digest); + /* Cache this error so we don't try to compute this one again. */ + status = CDM_DIFF_ERROR; + } + + for (u = 0; u < ARRAY_LENGTH(handles); ++u) { + compress_method_t method = compress_diffs_with[u]; + if (cache) { + cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, status, + handles[u]); + } else { + consensus_cache_entry_handle_free(handles[u]); + } + } + + consensus_diff_worker_job_free(job); +} + +/** + * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b> + * in a worker thread. + */ +static int +consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, + consensus_cache_entry_t *diff_to) +{ + tor_assert(in_main_thread()); + + consensus_cache_entry_incref(diff_from); + consensus_cache_entry_incref(diff_to); + + consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->diff_from = diff_from; + job->diff_to = diff_to; + + /* Make sure body is mapped. */ + const uint8_t *body; + size_t bodylen; + int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen); + int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen); + if (r1 < 0 || r2 < 0) + goto err; + + workqueue_entry_t *work; + work = cpuworker_queue_work(consensus_diff_worker_threadfn, + consensus_diff_worker_replyfn, + job); + if (!work) + goto err; + + return 0; + err: + consensus_diff_worker_job_free(job); // includes decrefs. + return -1; +} + diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h new file mode 100644 index 0000000000..048dae432c --- /dev/null +++ b/src/or/consdiffmgr.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONSDIFFMGR_H +#define TOR_CONSDIFFMGR_H + +/** + * Possible outcomes from trying to look up a given consensus diff. + */ +typedef enum consdiff_status_t { + CONSDIFF_AVAILABLE, + CONSDIFF_NOT_FOUND, + CONSDIFF_IN_PROGRESS, +} consdiff_status_t; + +typedef struct consdiff_cfg_t { + int32_t cache_max_age_hours; + int32_t cache_max_num; +} consdiff_cfg_t; + +struct consensus_cache_entry_t; // from conscache.h + +int consdiffmgr_add_consensus(const char *consensus, + const networkstatus_t *as_parsed); + +consdiff_status_t consdiffmgr_find_diff_from( + struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + int digest_type, + const uint8_t *digest, + size_t digestlen, + compress_method_t method); +void consdiffmgr_rescan(void); +int consdiffmgr_cleanup(void); +void consdiffmgr_configure(const consdiff_cfg_t *cfg); +struct sandbox_cfg_elem; +int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg); +void consdiffmgr_free_all(void); +int consdiffmgr_validate(void); + +#ifdef CONSDIFFMGR_PRIVATE +STATIC unsigned n_diff_compression_methods(void); +STATIC consensus_cache_t *cdm_cache_get(void); +STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( + consensus_flavor_t flavor, time_t valid_after); +STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, + consensus_cache_entry_t *ent, + const char *label); +STATIC int uncompress_or_copy(char **out, size_t *outlen, + consensus_cache_entry_t *ent); +#endif + +#endif + diff --git a/src/or/control.c b/src/or/control.c index 30067293dc..4e85c38123 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1873,7 +1873,7 @@ getinfo_helper_listeners(control_connection_t *control_conn, /** Implementation helper for GETINFO: knows the answers for questions about * directory information. */ -static int +STATIC int getinfo_helper_dir(control_connection_t *control_conn, const char *question, char **answer, const char **errmsg) @@ -2064,7 +2064,7 @@ getinfo_helper_dir(control_connection_t *control_conn, char d[DIGEST_LEN]; signed_descriptor_t *sd = NULL; if (base16_decode(d, sizeof(d), question, strlen(question)) - != sizeof(d)) { + == sizeof(d)) { /* XXXX this test should move into extrainfo_get_by_descriptor_digest, * but I don't want to risk affecting other parts of the code, * especially since the rules for using our own extrainfo (including @@ -3551,24 +3551,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, } /* Is this a single hop circuit? */ if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) { - const node_t *node = NULL; - char *exit_digest = NULL; - if (circ->build_state && - circ->build_state->chosen_exit && - !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { - exit_digest = circ->build_state->chosen_exit->identity_digest; - node = node_get_by_id(exit_digest); - } - /* Do both the client and relay allow one-hop exit circuits? */ - if (!node || - !node_allows_single_hop_exits(node) || - !get_options()->AllowSingleHopCircuits) { - connection_write_str_to_buf( - "551 Can't attach stream to this one-hop circuit.\r\n", conn); - return 0; - } - tor_assert(exit_digest); - ap_conn->chosen_exit_name = tor_strdup(hex_str(exit_digest, DIGEST_LEN)); + connection_write_str_to_buf( + "551 Can't attach stream to this one-hop circuit.\r\n", conn); + return 0; } if (circ && hop>0) { @@ -6924,6 +6909,11 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp) goto end; } + /* Without a directory fingerprint at this stage, we can't do much. */ + if (hsdir_fp == NULL) { + goto end; + } + /* OK, we have an onion address so now let's find which descriptor ID * is the one associated with the HSDir fingerprint. */ for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -7013,10 +7003,9 @@ control_event_hs_descriptor_receive_end(const char *action, char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; const char *desc_id = NULL; - if (!action || !id_digest || !rend_data || !onion_address) { - log_warn(LD_BUG, "Called with action==%p, id_digest==%p, " - "rend_data==%p, onion_address==%p", action, id_digest, - rend_data, onion_address); + if (!action || !rend_data || !onion_address) { + log_warn(LD_BUG, "Called with action==%p, rend_data==%p, " + "onion_address==%p", action, rend_data, onion_address); return; } @@ -7039,7 +7028,8 @@ control_event_hs_descriptor_receive_end(const char *action, rend_hsaddress_str_or_unknown(onion_address), rend_auth_type_to_string( TO_REND_DATA_V2(rend_data)->auth_type), - node_describe_longname_by_id(id_digest), + id_digest ? + node_describe_longname_by_id(id_digest) : "UNKNOWN", desc_id_field ? desc_id_field : "", reason_field ? reason_field : ""); @@ -7119,19 +7109,18 @@ control_event_hs_descriptor_uploaded(const char *id_digest, id_digest, NULL); } -/** Send HS_DESC event to inform controller that query <b>rend_query</b> - * failed to retrieve hidden service descriptor identified by - * <b>id_digest</b>. If <b>reason</b> is not NULL, add it to REASON= - * field. +/** Send HS_DESC event to inform controller that query <b>rend_data</b> + * failed to retrieve hidden service descriptor from directory identified by + * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL, + * add it to REASON= field. */ void control_event_hs_descriptor_failed(const rend_data_t *rend_data, const char *id_digest, const char *reason) { - if (!rend_data || !id_digest) { - log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p", - rend_data, id_digest); + if (!rend_data) { + log_warn(LD_BUG, "Called with rend_data==%p", rend_data); return; } control_event_hs_descriptor_receive_end("FAILED", @@ -7139,8 +7128,11 @@ control_event_hs_descriptor_failed(const rend_data_t *rend_data, rend_data, id_digest, reason); } -/** send HS_DESC_CONTENT event after completion of a successful fetch from - * hs directory. */ +/** Send HS_DESC_CONTENT event after completion of a successful fetch from hs + * directory. If <b>hsdir_id_digest</b> is NULL, it is replaced by "UNKNOWN". + * If <b>content</b> is NULL, it is replaced by an empty string. The + * <b>onion_address</b> or <b>desc_id</b> set to NULL will no trigger the + * control event. */ void control_event_hs_descriptor_content(const char *onion_address, const char *desc_id, @@ -7150,9 +7142,9 @@ control_event_hs_descriptor_content(const char *onion_address, static const char *event_name = "HS_DESC_CONTENT"; char *esc_content = NULL; - if (!onion_address || !desc_id || !hsdir_id_digest) { - log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, " - "hsdir_id_digest==%p", onion_address, desc_id, hsdir_id_digest); + if (!onion_address || !desc_id) { + log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ", + onion_address, desc_id); return; } @@ -7167,7 +7159,9 @@ control_event_hs_descriptor_content(const char *onion_address, event_name, rend_hsaddress_str_or_unknown(onion_address), desc_id, - node_describe_longname_by_id(hsdir_id_digest), + hsdir_id_digest ? + node_describe_longname_by_id(hsdir_id_digest) : + "UNKNOWN", esc_content); tor_free(esc_content); } diff --git a/src/or/control.h b/src/or/control.h index a786dfe1af..41a194bfcb 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -290,6 +290,10 @@ STATIC int getinfo_helper_downloads( control_connection_t *control_conn, const char *question, char **answer, const char **errmsg); +STATIC int getinfo_helper_dir( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); #endif diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 790c147d7b..af79fafaa6 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -479,6 +479,20 @@ queue_pending_tasks(void) } } +/** DOCDOC */ +MOCK_IMPL(workqueue_entry_t *, +cpuworker_queue_work,(workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg)) +{ + tor_assert(threadpool); + + return threadpool_queue_work(threadpool, + fn, + reply_fn, + arg); +} + /** Try to tell a cpuworker to perform the public key operations necessary to * respond to <b>onionskin</b> for the circuit <b>circ</b>. * diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h index fd426e21f6..aedf2fae32 100644 --- a/src/or/cpuworker.h +++ b/src/or/cpuworker.h @@ -14,6 +14,12 @@ void cpu_init(void); void cpuworkers_rotate_keyinfo(void); +struct workqueue_entry_s; +enum workqueue_reply_t; +MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, ( + enum workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg)); struct create_cell_t; int assign_onionskin_to_cpuworker(or_circuit_t *circ, diff --git a/src/or/directory.c b/src/or/directory.c index 1b999ee7c3..c1db40ef22 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -13,6 +13,8 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "consdiff.h" +#include "consdiffmgr.h" #include "control.h" #include "compat.h" #define DIRECTORY_PRIVATE @@ -63,7 +65,7 @@ * multi-hop circuits for anonymity. * * Directory requests are launched by calling - * directory_initiate_command_rend() or one of its numerous variants. This + * directory_initiate_request(). This * launch the connection, will construct an HTTP request with * directory_send_command(), send the and wait for a response. The client * later handles the response with connection_dir_client_reached_eof(), @@ -98,9 +100,8 @@ * connection_finished_connecting() in connection.c */ static void directory_send_command(dir_connection_t *conn, - int purpose, int direct, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); + int direct, + const directory_request_t *request); static int body_is_plausible(const char *body, size_t body_len, int purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); @@ -118,22 +119,10 @@ static void dir_microdesc_download_failed(smartlist_t *failed, int status_code); static int client_likes_consensus(networkstatus_t *v, const char *want_url); -static void directory_initiate_command_rend( - const tor_addr_port_t *or_addr_port, - const tor_addr_port_t *dir_addr_port, - const char *digest, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - circuit_guard_state_t *guard_state); - static void connection_dir_close_consensus_fetches( dir_connection_t *except_this_one, const char *resource); +static void directory_request_set_guard_state(directory_request_t *req, + struct circuit_guard_state_t *state); /********* START VARIABLES **********/ @@ -142,6 +131,7 @@ static void connection_dir_close_consensus_fetches( #define ALLOW_DIRECTORY_TIME_SKEW (30*60) #define X_ADDRESS_HEADER "X-Your-Address-Is: " +#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: " /** HTTP cache control: how long do we tell proxies they can cache each * kind of document we serve? */ @@ -421,11 +411,14 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, } else { indirection = DIRIND_DIRECT_CONN; } - directory_initiate_command_routerstatus(rs, dir_purpose, - router_purpose, - indirection, - NULL, payload, upload_len, 0, - NULL); + + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_payload(req, payload, upload_len); + directory_initiate_request(req); + directory_request_free(req); } SMARTLIST_FOREACH_END(ds); if (!found) { char *s = authdir_type_to_string(type); @@ -486,6 +479,70 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, return rs; } +/** + * Set the extra fields in <b>req</b> that are used when requesting a + * consensus of type <b>resource</b>. + * + * Right now, these fields are if-modified-since and x-or-diff-from-consensus. + */ +static void +dir_consensus_request_set_additional_headers(directory_request_t *req, + const char *resource) +{ + time_t if_modified_since = 0; + uint8_t or_diff_from[DIGEST256_LEN]; + int or_diff_from_is_set = 0; + + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ + const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180; + + int flav = FLAV_NS; + if (resource) + flav = networkstatus_parse_flavor_name(resource); + + if (flav != -1) { + /* IF we have a parsed consensus of this type, we can do an + * if-modified-time based on it. */ + networkstatus_t *v; + v = networkstatus_get_latest_consensus_by_flavor(flav); + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } else { + /* Otherwise it might be a consensus we don't parse, but which we + * do cache. Look at the cached copy, perhaps. */ + cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ + if (cd) { + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; + memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } + + if (if_modified_since > 0) + directory_request_set_if_modified_since(req, if_modified_since); + if (or_diff_from_is_set) { + char hex[HEX_DIGEST256_LEN + 1]; + base16_encode(hex, sizeof(hex), + (const char*)or_diff_from, sizeof(or_diff_from)); + directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex); + } +} + /** Start a connection to a random running directory server, using * connection purpose <b>dir_purpose</b>, intending to fetch descriptors * of purpose <b>router_purpose</b>, and requesting <b>resource</b>. @@ -507,47 +564,10 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose, resource); dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource); - time_t if_modified_since = 0; if (type == NO_DIRINFO) return; - if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - int flav = FLAV_NS; - networkstatus_t *v; - if (resource) - flav = networkstatus_parse_flavor_name(resource); - - /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus - * period of 1 hour. - */ -#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) - if (flav != -1) { - /* IF we have a parsed consensus of this type, we can do an - * if-modified-time based on it. */ - v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) { - /* In networks with particularly short V3AuthVotingIntervals, - * ask for the consensus if it's been modified since half the - * V3AuthVotingInterval of the most recent consensus. */ - time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; - if (v->fresh_until > v->valid_after - && ims_delay > (v->fresh_until - v->valid_after)/2) { - ims_delay = (v->fresh_until - v->valid_after)/2; - } - if_modified_since = v->valid_after + ims_delay; - } - } else { - /* Otherwise it might be a consensus we don't parse, but which we - * do cache. Look at the cached copy, perhaps. */ - cached_dir_t *cd = dirserv_get_consensus(resource); - /* We have no method of determining the voting interval from an - * unparsed consensus, so we use the default. */ - if (cd) - if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; - } - } - if (!options->FetchServerDescriptors) return; @@ -566,20 +586,20 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( routerinfo_t *ri = node->ri; /* clients always make OR connections to bridges */ tor_addr_port_t or_ap; - tor_addr_port_t nil_dir_ap; + directory_request_t *req = directory_request_new(dir_purpose); /* we are willing to use a non-preferred address if we need to */ fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &or_ap); - tor_addr_make_null(&nil_dir_ap.addr, AF_INET); - nil_dir_ap.port = 0; - directory_initiate_command_rend(&or_ap, - &nil_dir_ap, - ri->cache_info.identity_digest, - dir_purpose, - router_purpose, - DIRIND_ONEHOP, - resource, NULL, 0, if_modified_since, - NULL, guard_state); + directory_request_set_or_addr_port(req, &or_ap); + directory_request_set_directory_id_digest(req, + ri->cache_info.identity_digest); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); } else { if (guard_state) { entry_guard_cancel(&guard_state); @@ -639,12 +659,17 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( if (rs) { const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; - directory_initiate_command_routerstatus(rs, dir_purpose, - router_purpose, - indirection, - resource, NULL, 0, - if_modified_since, - guard_state); + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + if (guard_state) + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); } else { log_notice(LD_DIR, "While fetching directory info, " @@ -670,15 +695,17 @@ directory_get_from_all_authorities(uint8_t dir_purpose, SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(), dir_server_t *, ds) { - routerstatus_t *rs; if (router_digest_is_me(ds->digest)) continue; if (!(ds->type & V3_DIRINFO)) continue; - rs = &ds->fake_status; - directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose, - DIRIND_ONEHOP, resource, NULL, - 0, 0, NULL); + const routerstatus_t *rs = &ds->fake_status; + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); } SMARTLIST_FOREACH_END(ds); } @@ -778,110 +805,6 @@ directory_choose_address_routerstatus(const routerstatus_t *status, return 0; } -/** Same as directory_initiate_command_routerstatus(), but accepts - * rendezvous data to fetch a hidden service descriptor. */ -void -directory_initiate_command_routerstatus_rend(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - circuit_guard_state_t *guard_state) -{ - const or_options_t *options = get_options(); - const node_t *node; - tor_addr_port_t use_or_ap, use_dir_ap; - const int anonymized_connection = dirind_is_anon(indirection); - - tor_assert(status != NULL); - - node = node_get_by_id(status->identity_digest); - - /* XXX The below check is wrong: !node means it's not in the consensus, - * but we haven't checked if we have a descriptor for it -- and also, - * we only care about the descriptor if it's a begindir-style anonymized - * connection. */ - if (!node && anonymized_connection) { - log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " - "don't have its router descriptor.", - routerstatus_describe(status)); - return; - } - - if (options->ExcludeNodes && options->StrictNodes && - routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { - log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but " - "it's in our ExcludedNodes list and StrictNodes is set. " - "Skipping. This choice might make your Tor not work.", - routerstatus_describe(status), - dir_conn_purpose_to_string(dir_purpose)); - return; - } - - /* At this point, if we are a client making a direct connection to a - * directory server, we have selected a server that has at least one address - * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This - * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if - * possible. (If UseBridges is set, clients always use IPv6, and prefer it - * by default.) - * - * Now choose an address that we can use to connect to the directory server. - */ - if (directory_choose_address_routerstatus(status, indirection, &use_or_ap, - &use_dir_ap) < 0) { - return; - } - - /* We don't retry the alternate OR/Dir address for the same directory if - * the address we choose fails (#6772). - * Instead, we'll retry another directory on failure. */ - - directory_initiate_command_rend(&use_or_ap, &use_dir_ap, - status->identity_digest, - dir_purpose, router_purpose, - indirection, resource, - payload, payload_len, if_modified_since, - rend_query, - guard_state); -} - -/** Launch a new connection to the directory server <b>status</b> to - * upload or download a server or rendezvous - * descriptor. <b>dir_purpose</b> determines what - * kind of directory connection we're launching, and must be one of - * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b> - * specifies the descriptor purposes we have in mind (currently only - * used for FETCH_DIR). - * - * When uploading, <b>payload</b> and <b>payload_len</b> determine the content - * of the HTTP post. Otherwise, <b>payload</b> should be NULL. - * - * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we - * want to fetch. - */ -MOCK_IMPL(void, directory_initiate_command_routerstatus, - (const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - circuit_guard_state_t *guard_state)) -{ - directory_initiate_command_routerstatus_rend(status, dir_purpose, - router_purpose, - indirection, resource, - payload, payload_len, - if_modified_since, NULL, - guard_state); -} - /** Return true iff <b>conn</b> is the client side of a directory connection * we launched to ourself in order to determine the reachability of our * dir_port. */ @@ -1065,6 +988,47 @@ directory_must_use_begindir(const or_options_t *options) return !public_server_mode(options); } +struct directory_request_t { + /** + * These fields specify which directory we're contacting. Routerstatus, + * if present, overrides the other fields. + * + * @{ */ + tor_addr_port_t or_addr_port; + tor_addr_port_t dir_addr_port; + char digest[DIGEST_LEN]; + + const routerstatus_t *routerstatus; + /** @} */ + /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what + * kind of operation we'll be doing (upload/download), and of what kind + * of document. */ + uint8_t dir_purpose; + /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo + * and extrainfo docs. */ + uint8_t router_purpose; + /** Enum: determines whether to anonymize, and whether to use dirport or + * orport. */ + dir_indirection_t indirection; + /** Alias to the variable part of the URL for this request */ + const char *resource; + /** Alias to the payload to upload (if any) */ + const char *payload; + /** Number of bytes to upload from payload</b> */ + size_t payload_len; + /** Value to send in an if-modified-since header, or 0 for none. */ + time_t if_modified_since; + /** Hidden-service-specific information */ + const rend_data_t *rend_query; + /** Extra headers to append to the request */ + config_line_t *additional_headers; + /** */ + /** Used internally to directory.c: gets informed when the attempt to + * connect to the directory succeeds or fails, if that attempt bears on the + * directory's usability as a directory guard. */ + circuit_guard_state_t *guard_state; +}; + /** Evaluate the situation and decide if we should use an encrypted * "begindir-style" connection for this directory request. * 0) If there is no DirPort, yes. @@ -1078,12 +1042,16 @@ directory_must_use_begindir(const or_options_t *options) */ static int directory_command_should_use_begindir(const or_options_t *options, - const tor_addr_t *or_addr, int or_port, - const tor_addr_t *dir_addr, int dir_port, - dir_indirection_t indirection, + const directory_request_t *req, const char **reason) { - (void)dir_addr; + const tor_addr_t *or_addr = &req->or_addr_port.addr; + //const tor_addr_t *dir_addr = &req->dir_addr_port.addr; + const int or_port = req->or_addr_port.port; + const int dir_port = req->dir_addr_port.port; + + const dir_indirection_t indirection = req->indirection; + tor_assert(reason); *reason = NULL; @@ -1123,68 +1091,290 @@ directory_command_should_use_begindir(const or_options_t *options, return 1; } -/** Helper for directory_initiate_command_rend: send the - * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>, - * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose - * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and - * <b>router_purpose</b>, making an (in)direct connection as specified in - * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of - * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>. +/** + * Create and return a new directory_request_t with purpose + * <b>dir_purpose</b>. + */ +directory_request_t * +directory_request_new(uint8_t dir_purpose) +{ + tor_assert(dir_purpose >= DIR_PURPOSE_MIN_); + tor_assert(dir_purpose <= DIR_PURPOSE_MAX_); + tor_assert(dir_purpose != DIR_PURPOSE_SERVER); + tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2); + + directory_request_t *result = tor_malloc_zero(sizeof(*result)); + tor_addr_make_null(&result->or_addr_port.addr, AF_INET); + result->or_addr_port.port = 0; + tor_addr_make_null(&result->dir_addr_port.addr, AF_INET); + result->dir_addr_port.port = 0; + result->dir_purpose = dir_purpose; + result->router_purpose = ROUTER_PURPOSE_GENERAL; + result->indirection = DIRIND_ONEHOP; + return result; +} +/** + * Release all resources held by <b>req</b>. + */ +void +directory_request_free(directory_request_t *req) +{ + if (req == NULL) + return; + config_free_lines(req->additional_headers); + tor_free(req); +} +/** + * Set the address and OR port to use for this directory request. If there is + * no OR port, we'll have to connect over the dirport. (If there are both, + * the indirection setting determins which to use.) + */ +void +directory_request_set_or_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->or_addr_port, p, sizeof(*p)); +} +/** + * Set the address and dirport to use for this directory request. If there + * is no dirport, we'll have to connect over the OR port. (If there are both, + * the indirection setting determins which to use.) + */ +void +directory_request_set_dir_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->dir_addr_port, p, sizeof(*p)); +} +/** + * Set the RSA identity digest of the directory to use for this directory + * request. + */ +void +directory_request_set_directory_id_digest(directory_request_t *req, + const char *digest) +{ + memcpy(req->digest, digest, DIGEST_LEN); +} +/** + * Set the router purpose associated with uploaded and downloaded router + * descriptors and extrainfo documents in this directory request. The purpose + * must be one of ROUTER_PURPOSE_GENERAL (the default) or + * ROUTER_PURPOSE_BRIDGE. + */ +void +directory_request_set_router_purpose(directory_request_t *req, + uint8_t router_purpose) +{ + tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL || + router_purpose == ROUTER_PURPOSE_BRIDGE); + // assert that it actually makes sense to set this purpose, given + // the dir_purpose. + req->router_purpose = router_purpose; +} +/** + * Set the indirection to be used for the directory request. The indirection + * parameter configures whether to connect to a DirPort or ORPort, and whether + * to anonymize the connection. DIRIND_ONEHOP (use ORPort, don't anonymize) + * is the default. See dir_indirection_t for more information. + */ +void +directory_request_set_indirection(directory_request_t *req, + dir_indirection_t indirection) +{ + req->indirection = indirection; +} + +/** + * Set a pointer to the resource to request from a directory. Different + * request types use resources to indicate different components of their URL. + * Note that only an alias to <b>resource</b> is stored, so the + * <b>resource</b> must outlive the request. */ void -directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, - const tor_addr_t *dir_addr, uint16_t dir_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since) +directory_request_set_resource(directory_request_t *req, + const char *resource) { - tor_addr_port_t or_ap, dir_ap; + req->resource = resource; +} +/** + * Set a pointer to the payload to include with this directory request, along + * with its length. Note that only an alias to <b>payload</b> is stored, so + * the <b>payload</b> must outlive the request. + */ +void +directory_request_set_payload(directory_request_t *req, + const char *payload, + size_t payload_len) +{ + tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose)); - /* Use the null tor_addr and 0 port if the address or port isn't valid. */ - if (tor_addr_port_is_valid(or_addr, or_port, 0)) { - tor_addr_copy(&or_ap.addr, or_addr); - or_ap.port = or_port; - } else { - /* the family doesn't matter here, so make it IPv4 */ - tor_addr_make_null(&or_ap.addr, AF_INET); - or_ap.port = or_port = 0; + req->payload = payload; + req->payload_len = payload_len; +} +/** + * Set an if-modified-since date to send along with the request. The + * default is 0 (meaning, send no if-modified-since header). + */ +void +directory_request_set_if_modified_since(directory_request_t *req, + time_t if_modified_since) +{ + req->if_modified_since = if_modified_since; +} + +/** Include a header of name <b>key</b> with content <b>val</b> in the + * request. Neither may include newlines or other odd characters. Their + * ordering is not currently guaranteed. + * + * Note that, as elsewhere in this module, header keys include a trailing + * colon and space. + */ +void +directory_request_add_header(directory_request_t *req, + const char *key, + const char *val) +{ + config_line_prepend(&req->additional_headers, key, val); +} +/** + * Set an object containing HS data to be associated with this request. Note + * that only an alias to <b>query</b> is stored, so the <b>query</b> object + * must outlive the request. + */ +void +directory_request_set_rend_query(directory_request_t *req, + const rend_data_t *query) +{ + if (query) { + tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 || + req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); } + req->rend_query = query; +} +/** Set a static circuit_guard_state_t object to affliate with the request in + * <b>req</b>. This object will receive notification when the attempt to + * connect to the guard either succeeds or fails. */ +static void +directory_request_set_guard_state(directory_request_t *req, + circuit_guard_state_t *state) +{ + req->guard_state = state; +} - if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) { - tor_addr_copy(&dir_ap.addr, dir_addr); - dir_ap.port = dir_port; - } else { - /* the family doesn't matter here, so make it IPv4 */ - tor_addr_make_null(&dir_ap.addr, AF_INET); - dir_ap.port = dir_port = 0; +/** + * Internal: Return true if any information for contacting the directory in + * <b>req</b> has been set, other than by the routerstatus. */ +static int +directory_request_dir_contact_info_specified(const directory_request_t *req) +{ + /* We only check for ports here, since we don't use an addr unless the port + * is set */ + return (req->or_addr_port.port || + req->dir_addr_port.port || + ! tor_digest_is_zero(req->digest)); +} + +/** + * Set the routerstatus to use for the directory associated with this + * request. If this option is set, then no other function to set the + * directory's address or identity should be called. + */ +void +directory_request_set_routerstatus(directory_request_t *req, + const routerstatus_t *status) +{ + req->routerstatus = status; +} +/** + * Helper: update the addresses, ports, and identities in <b>req</b> + * from the routerstatus object in <b>req</b>. Return 0 on success. + * On failure, warn and return -1. + */ +static int +directory_request_set_dir_from_routerstatus(directory_request_t *req) + +{ + const routerstatus_t *status = req->routerstatus; + if (BUG(status == NULL)) + return -1; + const or_options_t *options = get_options(); + const node_t *node; + tor_addr_port_t use_or_ap, use_dir_ap; + const int anonymized_connection = dirind_is_anon(req->indirection); + + tor_assert(status != NULL); + + node = node_get_by_id(status->identity_digest); + + /* XXX The below check is wrong: !node means it's not in the consensus, + * but we haven't checked if we have a descriptor for it -- and also, + * we only care about the descriptor if it's a begindir-style anonymized + * connection. */ + if (!node && anonymized_connection) { + log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " + "don't have its router descriptor.", + routerstatus_describe(status)); + return -1; + } + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { + log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping. This choice might make your Tor not work.", + routerstatus_describe(status), + dir_conn_purpose_to_string(req->dir_purpose)); + return -1; + } + + /* At this point, if we are a client making a direct connection to a + * directory server, we have selected a server that has at least one address + * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This + * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if + * possible. (If UseBridges is set, clients always use IPv6, and prefer it + * by default.) + * + * Now choose an address that we can use to connect to the directory server. + */ + if (directory_choose_address_routerstatus(status, + req->indirection, &use_or_ap, + &use_dir_ap) < 0) { + return -1; } - directory_initiate_command_rend(&or_ap, &dir_ap, - digest, dir_purpose, - router_purpose, indirection, - resource, payload, payload_len, - if_modified_since, NULL, NULL); + directory_request_set_or_addr_port(req, &use_or_ap); + directory_request_set_dir_addr_port(req, &use_dir_ap); + directory_request_set_directory_id_digest(req, status->identity_digest); + return 0; } -/** Same as directory_initiate_command(), but accepts rendezvous data to - * fetch a hidden service descriptor, and takes its address & port arguments - * as tor_addr_port_t. */ -static void -directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, - const tor_addr_port_t *dir_addr_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - circuit_guard_state_t *guard_state) +/** + * Launch the provided directory request, configured in <b>request</b>. + * After this function is called, you can free <b>request</b>. + */ +MOCK_IMPL(void, +directory_initiate_request,(directory_request_t *request)) { - tor_assert(or_addr_port); - tor_assert(dir_addr_port); + tor_assert(request); + if (request->routerstatus) { + tor_assert_nonfatal( + ! directory_request_dir_contact_info_specified(request)); + if (directory_request_set_dir_from_routerstatus(request) < 0) { + return; + } + } + + const tor_addr_port_t *or_addr_port = &request->or_addr_port; + const tor_addr_port_t *dir_addr_port = &request->dir_addr_port; + const char *digest = request->digest; + const uint8_t dir_purpose = request->dir_purpose; + const uint8_t router_purpose = request->router_purpose; + const dir_indirection_t indirection = request->indirection; + const char *resource = request->resource; + const rend_data_t *rend_query = request->rend_query; + circuit_guard_state_t *guard_state = request->guard_state; + tor_assert(or_addr_port->port || dir_addr_port->port); tor_assert(digest); @@ -1194,11 +1384,8 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, const char *begindir_reason = NULL; /* Should the connection be to a relay's OR port (and inside that we will * send our directory request)? */ - const int use_begindir = directory_command_should_use_begindir(options, - &or_addr_port->addr, or_addr_port->port, - &dir_addr_port->addr, dir_addr_port->port, - indirection, - &begindir_reason); + const int use_begindir = + directory_command_should_use_begindir(options, request, &begindir_reason); /* Will the connection go via a three-hop Tor circuit? Note that this * is separate from whether it will use_begindir. */ @@ -1300,9 +1487,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, /* fall through */ case 0: /* queue the command on the outbuf */ - directory_send_command(conn, dir_purpose, 1, resource, - payload, payload_len, - if_modified_since); + directory_send_command(conn, 1, request); connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ @@ -1356,9 +1541,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, } conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; /* queue the command on the outbuf */ - directory_send_command(conn, dir_purpose, 0, resource, - payload, payload_len, - if_modified_since); + directory_send_command(conn, 0, request); connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(ENTRY_TO_CONN(linked_conn)); @@ -1461,15 +1644,22 @@ copy_ipv6_address(char* destination, const char* source, size_t len, } } -/** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command(). +/** Queue an appropriate HTTP command for <b>request</b> on + * <b>conn</b>-\>outbuf. If <b>direct</b> is true, we're making a + * non-anonymized connection to the dirport. */ static void directory_send_command(dir_connection_t *conn, - int purpose, int direct, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since) + const int direct, + const directory_request_t *req) { + tor_assert(req); + const int purpose = req->dir_purpose; + const char *resource = req->resource; + const char *payload = req->payload; + const size_t payload_len = req->payload_len; + const time_t if_modified_since = req->if_modified_since; + char proxystring[256]; char hoststring[128]; /* NEEDS to be the same size hoststring. @@ -1533,6 +1723,14 @@ directory_send_command(dir_connection_t *conn, proxystring[0] = 0; } + /* Add additional headers, if any */ + { + config_line_t *h; + for (h = req->additional_headers; h; h = h->next) { + smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value); + } + } + switch (purpose) { case DIR_PURPOSE_FETCH_CONSENSUS: /* resource is optional. If present, it's a flavor name */ @@ -1934,6 +2132,39 @@ load_downloaded_routers(const char *body, smartlist_t *which, return added; } +/** A structure to hold arguments passed into each directory response + * handler */ +typedef struct response_handler_args_t { + int status_code; + const char *reason; + const char *body; + size_t body_len; + const char *headers; +} response_handler_args_t; + +static int handle_response_fetch_consensus(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_certificate(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_status_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_detached_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_desc(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_microdesc(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_dir(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); + /** We are a client, and we've finished reading the server's * response. Parse it and act appropriately. * @@ -1959,8 +2190,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - time_t now = time(NULL); - int src_code; size_t received_bytes; received_bytes = connection_get_inbuf_len(TO_CONN(conn)); @@ -2054,6 +2283,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. I'll try again soon.", status_code, escaped(reason), conn->base_.address, conn->base_.port); + time_t now = approx_time(); if ((rs = router_get_mutable_consensus_status_by_id(id_digest))) rs->last_dir_503_at = now; if ((ds = router_get_fallback_dirserver_by_digest(id_digest))) @@ -2096,15 +2326,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } /* Try declared compression first if we can. */ if (compression == GZIP_METHOD || compression == ZLIB_METHOD) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, compression, + !allow_partial, LOG_PROTOCOL_WARN); /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ if (!new_body && (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && compression != guessed) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, guessed, + !allow_partial, LOG_PROTOCOL_WARN); /* If we're pretty sure that we have a compressed directory, and * we didn't manage to uncompress it, then warn and bail. */ if (!plausible && !new_body) { @@ -2121,474 +2351,735 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - int r; - const char *flavname = conn->requested_resource; - if (status_code != 200) { - int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; - tor_log(severity, LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching consensus directory.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - networkstatus_consensus_download_failed(status_code, flavname); - return -1; + int rv; + response_handler_args_t args; + memset(&args, 0, sizeof(args)); + args.status_code = status_code; + args.reason = reason; + args.body = body; + args.body_len = body_len; + args.headers = headers; + + switch (conn->base_.purpose) { + case DIR_PURPOSE_FETCH_CONSENSUS: + rv = handle_response_fetch_consensus(conn, &args); + break; + case DIR_PURPOSE_FETCH_CERTIFICATE: + rv = handle_response_fetch_certificate(conn, &args); + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + rv = handle_response_fetch_status_vote(conn, &args); + break; + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + rv = handle_response_fetch_detached_signatures(conn, &args); + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + rv = handle_response_fetch_desc(conn, &args); + break; + case DIR_PURPOSE_FETCH_MICRODESC: + rv = handle_response_fetch_microdesc(conn, &args); + break; + case DIR_PURPOSE_FETCH_RENDDESC_V2: + rv = handle_response_fetch_renddesc_v2(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_DIR: + rv = handle_response_upload_dir(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_SIGNATURES: + rv = handle_response_upload_signatures(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_VOTE: + rv = handle_response_upload_vote(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + rv = handle_response_upload_renddesc_v2(conn, &args); + break; + default: + tor_assert_nonfatal_unreached(); + rv = -1; + break; + } + tor_free(body); + tor_free(headers); + tor_free(reason); + return rv; +} + +/** + * Handler function: processes a response to a request for a networkstatus + * consensus document by checking the consensus, storing it, and marking + * router requests as reachable. + **/ +static int +handle_response_fetch_consensus(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS); + const int status_code = args->status_code; + const char *body = args->body; + const size_t body_len = args->body_len; + const char *reason = args->reason; + const time_t now = approx_time(); + + const char *consensus; + char *new_consensus = NULL; + const char *sourcename; + + int r; + const char *flavname = conn->requested_resource; + if (status_code != 200) { + int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; + tor_log(severity, LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching consensus directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + networkstatus_consensus_download_failed(status_code, flavname); + return -1; + } + + if (looks_like_a_consensus_diff(body, body_len)) { + /* First find our previous consensus. Maybe it's in ram, maybe not. */ + cached_dir_t *cd = dirserv_get_consensus(flavname); + const char *consensus_body; + char *owned_consensus = NULL; + if (cd) { + consensus_body = cd->dir; + } else { + owned_consensus = networkstatus_read_cached_consensus(flavname); + consensus_body = owned_consensus; } - log_info(LD_DIR,"Received consensus directory (body size %d) from server " - "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if ((r=networkstatus_set_current_consensus(body, flavname, 0, - conn->identity_digest))<0) { - log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, - "Unable to load %s consensus directory downloaded from " - "server '%s:%d'. I'll try again soon.", - flavname, conn->base_.address, conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); + if (!consensus_body) { + log_warn(LD_DIR, "Received a consensus diff, but we can't find " + "any %s-flavored consensus in our current cache.",flavname); networkstatus_consensus_download_failed(0, flavname); + // XXXX if this happens too much, see below return -1; } - /* If we launched other fetches for this consensus, cancel them. */ - connection_dir_close_consensus_fetches(conn, flavname); - - /* launches router downloads as needed */ - routers_update_all_from_networkstatus(now, 3); - update_microdescs_from_networkstatus(now); - update_microdesc_downloads(now); - directory_info_has_arrived(now, 0, 0); - if (authdir_mode_v3(get_options())) { - sr_act_post_consensus( - networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); + new_consensus = consensus_diff_apply(consensus_body, body); + tor_free(owned_consensus); + if (new_consensus == NULL) { + log_warn(LD_DIR, "Could not apply consensus diff received from server " + "'%s:%d'", conn->base_.address, conn->base_.port); + // XXXX If this happens too many times, we should maybe not use + // XXXX this directory for diffs any more? + networkstatus_consensus_download_failed(0, flavname); + return -1; } - log_info(LD_DIR, "Successfully loaded consensus."); + log_info(LD_DIR, "Applied consensus diff (size %d) from server " + "'%s:%d', resulting in a new consensus document (size %d).", + (int)body_len, conn->base_.address, conn->base_.port, + (int)strlen(new_consensus)); + consensus = new_consensus; + sourcename = "generated based on a diff"; + } else { + log_info(LD_DIR,"Received consensus directory (body size %d) from server " + "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); + consensus = body; + sourcename = "downloaded"; + } + + if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + conn->identity_digest))<0) { + log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, + "Unable to load %s consensus directory %s from " + "server '%s:%d'. I'll try again soon.", + flavname, sourcename, conn->base_.address, conn->base_.port); + networkstatus_consensus_download_failed(0, flavname); + tor_free(new_consensus); + return -1; } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) { - if (status_code != 200) { - log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/keys/%s\".", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource); - connection_dir_download_cert_failed(conn, status_code); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - log_info(LD_DIR,"Received authority certificates (body size %d) from " - "server '%s:%d'", - (int)body_len, conn->base_.address, conn->base_.port); + /* If we launched other fetches for this consensus, cancel them. */ + connection_dir_close_consensus_fetches(conn, flavname); - /* - * Tell trusted_dirs_load_certs_from_string() whether it was by fp - * or fp-sk pair. - */ - src_code = -1; - if (!strcmpstart(conn->requested_resource, "fp/")) { - src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST; - } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) { - src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST; - } + /* launches router downloads as needed */ + routers_update_all_from_networkstatus(now, 3); + update_microdescs_from_networkstatus(now); + update_microdesc_downloads(now); + directory_info_has_arrived(now, 0, 0); + if (authdir_mode_v3(get_options())) { + sr_act_post_consensus( + networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); + } + log_info(LD_DIR, "Successfully loaded consensus."); - if (src_code != -1) { - if (trusted_dirs_load_certs_from_string(body, src_code, 1, - conn->identity_digest)<0) { - log_warn(LD_DIR, "Unable to parse fetched certificates"); - /* if we fetched more than one and only some failed, the successful - * ones got flushed to disk so it's safe to call this on them */ - connection_dir_download_cert_failed(conn, status_code); - } else { - directory_info_has_arrived(now, 0, 0); - log_info(LD_DIR, "Successfully loaded certificates from fetch."); - } - } else { - log_warn(LD_DIR, - "Couldn't figure out what to do with fetched certificates for " - "unknown resource %s", - conn->requested_resource); + tor_free(new_consensus); + return 0; +} + +/** + * Handler function: processes a response to a request for one or more + * authority certificates + **/ +static int +handle_response_fetch_certificate(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + if (status_code != 200) { + log_warn(LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/keys/%s\".", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); + return -1; + } + log_info(LD_DIR,"Received authority certificates (body size %d) from " + "server '%s:%d'", + (int)body_len, conn->base_.address, conn->base_.port); + + /* + * Tell trusted_dirs_load_certs_from_string() whether it was by fp + * or fp-sk pair. + */ + int src_code = -1; + if (!strcmpstart(conn->requested_resource, "fp/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST; + } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST; + } + + if (src_code != -1) { + if (trusted_dirs_load_certs_from_string(body, src_code, 1, + conn->identity_digest)<0) { + log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); + } else { + time_t now = approx_time(); + directory_info_has_arrived(now, 0, 0); + log_info(LD_DIR, "Successfully loaded certificates from fetch."); } + } else { + log_warn(LD_DIR, + "Couldn't figure out what to do with fetched certificates for " + "unknown resource %s", + conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) { - const char *msg; - int st; - log_info(LD_DIR,"Got votes (body size %d) from server %s:%d", - (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - log_warn(LD_DIR, + return 0; +} + +/** + * Handler function: processes a response to a request for an authority's + * current networkstatus vote. + **/ +static int +handle_response_fetch_status_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg; + int st; + log_info(LD_DIR,"Got votes (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, "Received http status code %d (%s) from server " "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".", status_code, escaped(reason), conn->base_.address, conn->base_.port, conn->requested_resource); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - dirvote_add_vote(body, &msg, &st); - if (st > 299) { - log_warn(LD_DIR, "Error adding retrieved vote: %s", msg); - } else { - log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg); - } + return -1; } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { - const char *msg = NULL; - log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d", - (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - log_warn(LD_DIR, + dirvote_add_vote(body, &msg, &st); + if (st > 299) { + log_warn(LD_DIR, "Error adding retrieved vote: %s", msg); + } else { + log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg); + } + + return 0; +} + +/** + * Handler function: processes a response to a request for the signatures + * that an authority knows about on a given consensus. + **/ +static int +handle_response_fetch_detached_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg = NULL; + log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, "Received http status code %d (%s) from server '%s:%d' while fetching " "\"/tor/status-vote/next/consensus-signatures.z\".", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) { - log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s", - conn->base_.address, conn->base_.port, msg?msg:"???"); - } + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + return -1; + } + if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) { + log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s", + conn->base_.address, conn->base_.port, msg?msg:"???"); } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || - conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { - int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO; - smartlist_t *which = NULL; - int n_asked_for = 0; - int descriptor_digests = conn->requested_resource && - !strcmpstart(conn->requested_resource,"d/"); - log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'", - was_ei ? "extra server info" : "server info", - (int)body_len, conn->base_.address, conn->base_.port); - if (conn->requested_resource && - (!strcmpstart(conn->requested_resource,"d/") || - !strcmpstart(conn->requested_resource,"fp/"))) { - which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource + - (descriptor_digests ? 2 : 3), - which, NULL, 0); - n_asked_for = smartlist_len(which); - } - if (status_code != 200) { - int dir_okay = status_code == 404 || - (status_code == 400 && !strcmp(reason, "Servers unavailable.")); - /* 404 means that it didn't have them; no big deal. - * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */ - log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR, - "Received http status code %d (%s) from server '%s:%d' " - "while fetching \"/tor/server/%s\". I'll try again soon.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource); - if (!which) { - connection_dir_download_routerdesc_failed(conn); - } else { - dir_routerdesc_download_failed(which, status_code, - conn->router_purpose, - was_ei, descriptor_digests); - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - } - tor_free(body); tor_free(headers); tor_free(reason); - return dir_okay ? 0 : -1; - } - /* Learn the routers, assuming we requested by fingerprint or "all" - * or "authority". - * - * We use "authority" to fetch our own descriptor for - * testing, and to fetch bridge descriptors for bootstrapping. Ignore - * the output of "authority" requests unless we are using bridges, - * since otherwise they'll be the response from reachability tests, - * and we don't really want to add that to our routerlist. */ - if (which || (conn->requested_resource && - (!strcmpstart(conn->requested_resource, "all") || - (!strcmpstart(conn->requested_resource, "authority") && - get_options()->UseBridges)))) { - /* as we learn from them, we remove them from 'which' */ - if (was_ei) { - router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which, - descriptor_digests); - } else { - //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, - // descriptor_digests, conn->router_purpose); - if (load_downloaded_routers(body, which, descriptor_digests, - conn->router_purpose, - conn->base_.address)) - directory_info_has_arrived(now, 0, 0); - } - } - if (which) { /* mark remaining ones as failed */ - log_info(LD_DIR, "Received %d/%d %s requested from %s:%d", - n_asked_for-smartlist_len(which), n_asked_for, - was_ei ? "extra-info documents" : "router descriptors", - conn->base_.address, (int)conn->base_.port); - if (smartlist_len(which)) { - dir_routerdesc_download_failed(which, status_code, - conn->router_purpose, - was_ei, descriptor_digests); - } - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - } - if (directory_conn_is_self_reachability_test(conn)) - router_dirport_found_reachable(); - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) { - smartlist_t *which = NULL; - log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " - "body size %d) from server '%s:%d'", - status_code, (int)body_len, conn->base_.address, - conn->base_.port); - tor_assert(conn->requested_resource && - !strcmpstart(conn->requested_resource, "d/")); + return 0; +} + +/** + * Handler function: processes a response to a request for a group of server + * descriptors or an extrainfo documents. + **/ +static int +handle_response_fetch_desc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO; + smartlist_t *which = NULL; + int n_asked_for = 0; + int descriptor_digests = conn->requested_resource && + !strcmpstart(conn->requested_resource,"d/"); + log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'", + was_ei ? "extra server info" : "server info", + (int)body_len, conn->base_.address, conn->base_.port); + if (conn->requested_resource && + (!strcmpstart(conn->requested_resource,"d/") || + !strcmpstart(conn->requested_resource,"fp/"))) { which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+2, - which, NULL, - DSR_DIGEST256|DSR_BASE64); - if (status_code != 200) { - log_info(LD_DIR, "Received status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " - "soon.", - status_code, escaped(reason), conn->base_.address, - (int)conn->base_.port, conn->requested_resource); - dir_microdesc_download_failed(which, status_code); + dir_split_resource_into_fingerprints(conn->requested_resource + + (descriptor_digests ? 2 : 3), + which, NULL, 0); + n_asked_for = smartlist_len(which); + } + if (status_code != 200) { + int dir_okay = status_code == 404 || + (status_code == 400 && !strcmp(reason, "Servers unavailable.")); + /* 404 means that it didn't have them; no big deal. + * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */ + log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR, + "Received http status code %d (%s) from server '%s:%d' " + "while fetching \"/tor/server/%s\". I'll try again soon.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + if (!which) { + connection_dir_download_routerdesc_failed(conn); + } else { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); smartlist_free(which); - tor_free(body); tor_free(headers); tor_free(reason); - return 0; + } + return dir_okay ? 0 : -1; + } + /* Learn the routers, assuming we requested by fingerprint or "all" + * or "authority". + * + * We use "authority" to fetch our own descriptor for + * testing, and to fetch bridge descriptors for bootstrapping. Ignore + * the output of "authority" requests unless we are using bridges, + * since otherwise they'll be the response from reachability tests, + * and we don't really want to add that to our routerlist. */ + if (which || (conn->requested_resource && + (!strcmpstart(conn->requested_resource, "all") || + (!strcmpstart(conn->requested_resource, "authority") && + get_options()->UseBridges)))) { + /* as we learn from them, we remove them from 'which' */ + if (was_ei) { + router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which, + descriptor_digests); } else { - smartlist_t *mds; - mds = microdescs_add_to_cache(get_microdesc_cache(), - body, body+body_len, SAVED_NOWHERE, 0, - now, which); - if (smartlist_len(which)) { - /* Mark remaining ones as failed. */ - dir_microdesc_download_failed(which, status_code); - } - if (mds && smartlist_len(mds)) { - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); - directory_info_has_arrived(now, 0, 1); + //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, + // descriptor_digests, conn->router_purpose); + if (load_downloaded_routers(body, which, descriptor_digests, + conn->router_purpose, + conn->base_.address)) { + time_t now = approx_time(); + directory_info_has_arrived(now, 0, 0); } - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - smartlist_free(mds); } } + if (which) { /* mark remaining ones as failed */ + log_info(LD_DIR, "Received %d/%d %s requested from %s:%d", + n_asked_for-smartlist_len(which), n_asked_for, + was_ei ? "extra-info documents" : "router descriptors", + conn->base_.address, (int)conn->base_.port); + if (smartlist_len(which)) { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); + } + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + } + if (directory_conn_is_self_reachability_test(conn)) + router_dirport_found_reachable(); - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR) { - switch (status_code) { - case 200: { - dir_server_t *ds = - router_get_trusteddirserver_by_digest(conn->identity_digest); - char *rejected_hdr = http_get_header(headers, - "X-Descriptor-Not-New: "); - if (rejected_hdr) { - if (!strcmp(rejected_hdr, "Yes")) { - log_info(LD_GENERAL, - "Authority '%s' declined our descriptor (not new)", - ds->nickname); - /* XXXX use this information; be sure to upload next one - * sooner. -NM */ - /* XXXX++ On further thought, the task above implies that we're - * basing our regenerate-descriptor time on when we uploaded the - * last descriptor, not on the published time of the last - * descriptor. If those are different, that's a bad thing to - * do. -NM */ - } - tor_free(rejected_hdr); - } - log_info(LD_GENERAL,"eof (status 200) after uploading server " - "descriptor: finished."); - control_event_server_status( - LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d", - conn->base_.address, conn->base_.port); - - ds->has_accepted_serverdesc = 1; - if (directories_have_accepted_server_descriptor()) - control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR"); - } - break; - case 400: - log_warn(LD_GENERAL,"http status 400 (%s) response from " - "dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - control_event_server_status(LOG_WARN, - "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"", - conn->base_.address, conn->base_.port, escaped(reason)); - break; - default: - log_warn(LD_GENERAL, + return 0; +} + +/** + * Handler function: processes a response to a request for a group of + * microdescriptors + **/ +static int +handle_response_fetch_microdesc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + smartlist_t *which = NULL; + log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " + "body size %d) from server '%s:%d'", + status_code, (int)body_len, conn->base_.address, + conn->base_.port); + tor_assert(conn->requested_resource && + !strcmpstart(conn->requested_resource, "d/")); + which = smartlist_new(); + dir_split_resource_into_fingerprints(conn->requested_resource+2, + which, NULL, + DSR_DIGEST256|DSR_BASE64); + if (status_code != 200) { + log_info(LD_DIR, "Received status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " + "soon.", + status_code, escaped(reason), conn->base_.address, + (int)conn->base_.port, conn->requested_resource); + dir_microdesc_download_failed(which, status_code); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + return 0; + } else { + smartlist_t *mds; + time_t now = approx_time(); + mds = microdescs_add_to_cache(get_microdesc_cache(), + body, body+body_len, SAVED_NOWHERE, 0, + now, which); + if (smartlist_len(which)) { + /* Mark remaining ones as failed. */ + dir_microdesc_download_failed(which, status_code); + } + if (mds && smartlist_len(mds)) { + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); + directory_info_has_arrived(now, 0, 1); + } + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + smartlist_free(mds); + } + + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload our + * router descriptor. + **/ +static int +handle_response_upload_dir(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *headers = args->headers; + + switch (status_code) { + case 200: { + dir_server_t *ds = + router_get_trusteddirserver_by_digest(conn->identity_digest); + char *rejected_hdr = http_get_header(headers, + "X-Descriptor-Not-New: "); + if (rejected_hdr) { + if (!strcmp(rejected_hdr, "Yes")) { + log_info(LD_GENERAL, + "Authority '%s' declined our descriptor (not new)", + ds->nickname); + /* XXXX use this information; be sure to upload next one + * sooner. -NM */ + /* XXXX++ On further thought, the task above implies that we're + * basing our regenerate-descriptor time on when we uploaded the + * last descriptor, not on the published time of the last + * descriptor. If those are different, that's a bad thing to + * do. -NM */ + } + tor_free(rejected_hdr); + } + log_info(LD_GENERAL,"eof (status 200) after uploading server " + "descriptor: finished."); + control_event_server_status( + LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d", + conn->base_.address, conn->base_.port); + + ds->has_accepted_serverdesc = 1; + if (directories_have_accepted_server_descriptor()) + control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR"); + } + break; + case 400: + log_warn(LD_GENERAL,"http status 400 (%s) response from " + "dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + control_event_server_status(LOG_WARN, + "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"", + conn->base_.address, conn->base_.port, escaped(reason)); + break; + default: + log_warn(LD_GENERAL, "http status %d (%s) reason unexpected while uploading " "descriptor to server '%s:%d').", status_code, escaped(reason), conn->base_.address, conn->base_.port); - break; - } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ + break; } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE) { - switch (status_code) { - case 200: { - log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d", - conn->base_.address, conn->base_.port); - } - break; - case 400: - log_warn(LD_DIR,"http status 400 (%s) response after uploading " - "vote to dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - break; - default: - log_warn(LD_GENERAL, + return 0; +} + +/** + * Handler function: processes a response to POST request to upload our + * own networkstatus vote. + **/ +static int +handle_response_upload_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "vote to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, "http status %d (%s) reason unexpected while uploading " "vote to server '%s:%d').", status_code, escaped(reason), conn->base_.address, conn->base_.port); - break; - } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ + break; } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + return 0; +} - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES) { - switch (status_code) { - case 200: { - log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d", - conn->base_.address, conn->base_.port); - } - break; - case 400: - log_warn(LD_DIR,"http status 400 (%s) response after uploading " - "signatures to dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - break; - default: - log_warn(LD_GENERAL, +/** + * Handler function: processes a response to POST request to upload our + * view of the signatures on the current consensus. + **/ +static int +handle_response_upload_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "signatures to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, "http status %d (%s) reason unexpected while uploading " "signatures to server '%s:%d').", status_code, escaped(reason), conn->base_.address, conn->base_.port); - break; - } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ - } - - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { - #define SEND_HS_DESC_FAILED_EVENT(reason) ( \ - control_event_hs_descriptor_failed(conn->rend_data, \ - conn->identity_digest, \ - reason) ) - #define SEND_HS_DESC_FAILED_CONTENT() ( \ - control_event_hs_descriptor_content(rend_data_get_address(conn->rend_data), \ - conn->requested_resource, \ + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + + return 0; +} + +/** + * Handler function: processes a response to a request for a v2 hidden service + * descriptor. + **/ +static int +handle_response_fetch_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + +#define SEND_HS_DESC_FAILED_EVENT(reason) \ + (control_event_hs_descriptor_failed(conn->rend_data, \ conn->identity_digest, \ - NULL) ) - tor_assert(conn->rend_data); - log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d " - "(%s))", - (int)body_len, status_code, escaped(reason)); - switch (status_code) { - case 200: - { - rend_cache_entry_t *entry = NULL; - - if (rend_cache_store_v2_desc_as_client(body, - conn->requested_resource, conn->rend_data, &entry) < 0) { - log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " - "Retrying at another directory."); - /* We'll retry when connection_about_to_close_connection() - * cleans this dir conn up. */ - SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); - SEND_HS_DESC_FAILED_CONTENT(); - } else { - char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; - /* Should never be NULL here if we found the descriptor. */ - tor_assert(entry); - rend_get_service_id(entry->parsed->pk, service_id); - - /* success. notify pending connections about this. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor."); - control_event_hs_descriptor_received(service_id, - conn->rend_data, - conn->identity_digest); - control_event_hs_descriptor_content(service_id, - conn->requested_resource, - conn->identity_digest, - body); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; - rend_client_desc_trynow(service_id); - memwipe(service_id, 0, sizeof(service_id)); - } - break; - } - case 404: - /* Not there. We'll retry when - * connection_about_to_close_connection() cleans this conn up. */ - log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " - "Retrying at another directory."); - SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); - SEND_HS_DESC_FAILED_CONTENT(); - break; - case 400: - log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " - "http status 400 (%s). Dirserver didn't like our " - "v2 rendezvous query? Retrying at another directory.", - escaped(reason)); - SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); - SEND_HS_DESC_FAILED_CONTENT(); - break; - default: - log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " - "http status %d (%s) response unexpected while " - "fetching v2 hidden service descriptor (server '%s:%d'). " - "Retrying at another directory.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + reason)) +#define SEND_HS_DESC_FAILED_CONTENT() \ + (control_event_hs_descriptor_content( \ + rend_data_get_address(conn->rend_data), \ + conn->requested_resource, \ + conn->identity_digest, \ + NULL)) + + tor_assert(conn->rend_data); + log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d " + "(%s))", + (int)body_len, status_code, escaped(reason)); + switch (status_code) { + case 200: + { + rend_cache_entry_t *entry = NULL; + + if (rend_cache_store_v2_desc_as_client(body, + conn->requested_resource, + conn->rend_data, &entry) < 0) { + log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " + "Retrying at another directory."); + /* We'll retry when connection_about_to_close_connection() + * cleans this dir conn up. */ + SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); SEND_HS_DESC_FAILED_CONTENT(); - break; + } else { + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + /* Should never be NULL here if we found the descriptor. */ + tor_assert(entry); + rend_get_service_id(entry->parsed->pk, service_id); + + /* success. notify pending connections about this. */ + log_info(LD_REND, "Successfully fetched v2 rendezvous " + "descriptor."); + control_event_hs_descriptor_received(service_id, + conn->rend_data, + conn->identity_digest); + control_event_hs_descriptor_content(service_id, + conn->requested_resource, + conn->identity_digest, + body); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; + rend_client_desc_trynow(service_id); + memwipe(service_id, 0, sizeof(service_id)); + } + break; } + case 404: + /* Not there. We'll retry when + * connection_about_to_close_connection() cleans this conn up. */ + log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " + "Retrying at another directory."); + SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + case 400: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status 400 (%s). Dirserver didn't like our " + "v2 rendezvous query? Retrying at another directory.", + escaped(reason)); + SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + default: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status %d (%s) response unexpected while " + "fetching v2 hidden service descriptor (server '%s:%d'). " + "Retrying at another directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { - #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \ - control_event_hs_descriptor_upload_failed( \ - conn->identity_digest, \ - rend_data_get_address(conn->rend_data), \ - reason) ) - log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " - "(%s))", - status_code, escaped(reason)); - /* Without the rend data, we'll have a problem identifying what has been - * uploaded for which service. */ - tor_assert(conn->rend_data); - switch (status_code) { - case 200: - log_info(LD_REND, - "Uploading rendezvous descriptor: finished with status " - "200 (%s)", escaped(reason)); - control_event_hs_descriptor_uploaded(conn->identity_digest, - rend_data_get_address(conn->rend_data)); - rend_service_desc_has_uploaded(conn->rend_data); - break; - case 400: - log_warn(LD_REND,"http status 400 (%s) response from dirserver " - "'%s:%d'. Malformed rendezvous descriptor?", - escaped(reason), conn->base_.address, conn->base_.port); - SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); - break; - default: - log_warn(LD_REND,"http status %d (%s) response unexpected (server " - "'%s:%d').", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); - break; - } + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload a v2 + * hidden service descriptor. + **/ +static int +handle_response_upload_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + +#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) \ + (control_event_hs_descriptor_upload_failed( \ + conn->identity_digest, \ + rend_data_get_address(conn->rend_data), \ + reason)) + + log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " + "(%s))", + status_code, escaped(reason)); + /* Without the rend data, we'll have a problem identifying what has been + * uploaded for which service. */ + tor_assert(conn->rend_data); + switch (status_code) { + case 200: + log_info(LD_REND, + "Uploading rendezvous descriptor: finished with status " + "200 (%s)", escaped(reason)); + control_event_hs_descriptor_uploaded(conn->identity_digest, + rend_data_get_address(conn->rend_data)); + rend_service_desc_has_uploaded(conn->rend_data); + break; + case 400: + log_warn(LD_REND,"http status 400 (%s) response from dirserver " + "'%s:%d'. Malformed rendezvous descriptor?", + escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); + break; + default: + log_warn(LD_REND,"http status %d (%s) response unexpected (server " + "'%s:%d').", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); + break; } - tor_free(body); tor_free(headers); tor_free(reason); + return 0; } @@ -2780,13 +3271,54 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, * based on whether the response will be <b>compressed</b> or not. */ static void write_http_response_header(dir_connection_t *conn, ssize_t length, - int compressed, long cache_lifetime) + compress_method_t method, long cache_lifetime) { + const char *methodname = compression_method_get_name(method); + const char *doctype; + if (method == NO_METHOD) + doctype = "text/plain"; + else + doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - compressed?"application/octet-stream":"text/plain", - compressed?"deflate":"identity", - NULL, - cache_lifetime); + doctype, + methodname, + NULL, + cache_lifetime); +} + +/** Array of compression methods to use (if supported) for serving + * precompressed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_precompressed[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>, + * and convert them to a bitfield where compression method x is supported if + * and only if 1 << x is set in the bitfield. */ +STATIC unsigned +parse_accept_encoding_header(const char *h) +{ + unsigned result = (1u << NO_METHOD); + smartlist_t *methods = smartlist_new(); + smartlist_split_string(methods, h, ",", + SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + SMARTLIST_FOREACH_BEGIN(methods, const char *, m) { + compress_method_t method = compression_method_get_by_name(m); + if (method != UNKNOWN_METHOD) { + tor_assert(((unsigned)method) < 8*sizeof(unsigned)); + result |= (1u << method); + } + } SMARTLIST_FOREACH_END(m); + SMARTLIST_FOREACH_BEGIN(methods, char *, m) { + tor_free(m); + } SMARTLIST_FOREACH_END(m); + smartlist_free(methods); + return result; } /** Decide whether a client would accept the consensus we have. @@ -2845,7 +3377,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) /** Return the compression level we should use for sending a compressed * response of size <b>n_bytes</b>. */ -STATIC zlib_compression_level_t +STATIC compression_level_t choose_compression_level(ssize_t n_bytes) { if (! have_been_under_memory_pressure()) { @@ -2863,8 +3395,9 @@ choose_compression_level(ssize_t n_bytes) /** Information passed to handle a GET request. */ typedef struct get_handler_args_t { - /** True if the client asked for compressed data. */ - int compressed; + /** Bitmask of compression methods that the client said (or implied) it + * supported. */ + unsigned compression_supported; /** If nonzero, the time included an if-modified-since header with this * value. */ time_t if_modified_since; @@ -2938,8 +3471,9 @@ directory_handle_command_get,(dir_connection_t *conn, const char *headers, { char *url, *url_mem, *header; time_t if_modified_since = 0; - int compressed; + int zlib_compressed_in_url; size_t url_len; + unsigned compression_methods_supported; /* We ignore the body of a GET request. */ (void)req_body; @@ -2970,17 +3504,31 @@ directory_handle_command_get,(dir_connection_t *conn, const char *headers, url_mem = url; url_len = strlen(url); - compressed = url_len > 2 && !strcmp(url+url_len-2, ".z"); - if (compressed) { + + zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z"); + if (zlib_compressed_in_url) { url[url_len-2] = '\0'; url_len -= 2; } + if ((header = http_get_header(headers, "Accept-Encoding: "))) { + compression_methods_supported = parse_accept_encoding_header(header); + tor_free(header); + } else { + compression_methods_supported = (1u << NO_METHOD); + } + if (zlib_compressed_in_url) { + compression_methods_supported |= (1u << ZLIB_METHOD); + } + + /* Remove all methods that we don't both support. */ + compression_methods_supported &= tor_compress_get_supported_method_bitmask(); + get_handler_args_t args; args.url = url; args.headers = headers; args.if_modified_since = if_modified_since; - args.compressed = compressed; + args.compression_supported = compression_methods_supported; int i, result = -1; for (i = 0; url_table[i].string; ++i) { @@ -3052,6 +3600,70 @@ warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) } } +/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>, + * set <b>digest_out<b> to a new smartlist containing every 256-bit + * hex-encoded digest listed in that header and return 0. Otherwise return + * -1. */ +static int +parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) +{ + char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER); + if (hdr == NULL) { + return -1; + } + smartlist_t *hex_digests = smartlist_new(); + *digests_out = smartlist_new(); + smartlist_split_string(hex_digests, hdr, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) { + uint8_t digest[DIGEST256_LEN]; + if (base16_decode((char*)digest, sizeof(digest), hex, strlen(hex)) == + DIGEST256_LEN) { + smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest))); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "X-Or-Diff-From-Consensus header contained bogus digest %s; " + "ignoring.", escaped(hex)); + } + } SMARTLIST_FOREACH_END(hex); + SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp)); + smartlist_free(hex_digests); + tor_free(hdr); + return 0; +} + +/** + * Try to find the best consensus diff possible in order to serve a client + * request for a diff from one of the consensuses in <b>digests</b> to the + * current consensus of flavor <b>flav</b>. The client supports the + * compression methods listed in the <b>compression_methods</b> bitfield: + * place the method chosen (if any) into <b>compression_used_out</b>. + */ +static struct consensus_cache_entry_t * +find_best_diff(const smartlist_t *digests, int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + unsigned u; + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + if (0 == (compression_methods & (1u<<method))) + continue; // client doesn't like this one, or we don't have it. + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, + diff_from, DIGEST256_LEN, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; + } + } + } SMARTLIST_FOREACH_END(diff_from); + return NULL; +} + /** Helper function for GET /tor/status-vote/current/consensus */ static int @@ -3059,133 +3671,150 @@ handle_get_current_consensus(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const int compressed = args->compression_supported & (1u << ZLIB_METHOD); const time_t if_modified_since = args->if_modified_since; int clear_spool = 0; - { - /* v3 network status fetch. */ - long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + /* v3 network status fetch. */ + long lifetime = NETWORKSTATUS_CACHE_LIFETIME; - networkstatus_t *v; - time_t now = time(NULL); - const char *want_fps = NULL; - char *flavor = NULL; - int flav = FLAV_NS; + networkstatus_t *v; + time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; + int flav = FLAV_NS; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" - /* figure out the flavor if any, and who we wanted to sign the thing */ - if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { - const char *f, *cp; - f = url + strlen(CONSENSUS_FLAVORED_PREFIX); - cp = strchr(f, '/'); - if (cp) { - want_fps = cp+1; - flavor = tor_strndup(f, cp-f); - } else { - flavor = tor_strdup(f); - } - flav = networkstatus_parse_flavor_name(flavor); - if (flav < 0) - flav = FLAV_NS; + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); } else { - if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) - want_fps = url+strlen(CONSENSUS_URL_PREFIX); - } - - v = networkstatus_get_latest_consensus_by_flavor(flav); - - if (v && !networkstatus_consensus_reasonably_live(v, now)) { - write_http_status_line(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(v, flavor, now); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - tor_free(flavor); - goto done; + flavor = tor_strdup(f); } + flav = networkstatus_parse_flavor_name(flavor); + if (flav < 0) + flav = FLAV_NS; + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } - if (v && want_fps && - !client_likes_consensus(v, want_fps)) { - write_http_status_line(conn, 404, "Consensus not signed by sufficient " - "number of requested authorities"); - geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); - tor_free(flavor); - goto done; - } + v = networkstatus_get_latest_consensus_by_flavor(flav); - conn->spool = smartlist_new(); - clear_spool = 1; - { - spooled_resource_t *spooled; - if (flavor) - spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, - (uint8_t*)flavor, strlen(flavor)); - else - spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, - NULL, 0); - tor_free(flavor); - smartlist_add(conn->spool, spooled); - } - lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; + if (v && !networkstatus_consensus_reasonably_live(v, now)) { + write_http_status_line(conn, 404, "Consensus is too old"); + warn_consensus_is_too_old(v, flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + tor_free(flavor); + goto done; + } - if (!smartlist_len(conn->spool)) { /* we failed to create/cache cp */ - write_http_status_line(conn, 503, "Network status object unavailable"); - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); - goto done; - } + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { + write_http_status_line(conn, 404, "Consensus not signed by sufficient " + "number of requested authorities"); + geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); + goto done; + } - size_t size_guess = 0; - int n_expired = 0; - dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, - compressed, - &size_guess, - &n_expired); + struct consensus_cache_entry_t *cached_diff = NULL; + smartlist_t *diff_from_digests = NULL; + compress_method_t compression_used = NO_METHOD; + if (!parse_or_diff_from_header(&diff_from_digests, args->headers)) { + tor_assert(diff_from_digests); + cached_diff = find_best_diff(diff_from_digests, flav, + args->compression_supported, + &compression_used); + SMARTLIST_FOREACH(diff_from_digests, uint8_t *, d, tor_free(d)); + smartlist_free(diff_from_digests); + } - if (!smartlist_len(conn->spool) && !n_expired) { - write_http_status_line(conn, 404, "Not found"); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - goto done; - } else if (!smartlist_len(conn->spool)) { - write_http_status_line(conn, 304, "Not modified"); - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); - goto done; + conn->spool = smartlist_new(); + clear_spool = 1; + { + spooled_resource_t *spooled; + if (cached_diff) { + spooled = spooled_resource_new_from_cache_entry(cached_diff); + } else if (flavor) { + spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, + (uint8_t*)flavor, strlen(flavor)); + compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + } else { + spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, + NULL, 0); + compression_used = compressed ? ZLIB_METHOD : NO_METHOD; } + tor_free(flavor); + smartlist_add(conn->spool, spooled); + } + lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; - if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { - log_debug(LD_DIRSERV, - "Client asked for network status lists, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - geoip_note_ns_response(GEOIP_REJECT_BUSY); - goto done; - } + if (!smartlist_len(conn->spool)) { /* we failed to create/cache cp */ + write_http_status_line(conn, 503, "Network status object unavailable"); + geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); + goto done; + } - tor_addr_t addr; - if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, - &addr, NULL, - time(NULL)); - geoip_note_ns_response(GEOIP_SUCCESS); - /* Note that a request for a network status has started, so that we - * can measure the download time later on. */ - if (conn->dirreq_id) - geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED); - else - geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess, - DIRREQ_DIRECT); - } + size_t size_guess = 0; + int n_expired = 0; + dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, + compressed, + &size_guess, + &n_expired); - clear_spool = 0; - write_http_response_header(conn, -1, compressed, - smartlist_len(conn->spool) == 1 ? lifetime : 0); - if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + if (!smartlist_len(conn->spool) && !n_expired) { + write_http_status_line(conn, 404, "Not found"); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } else if (!smartlist_len(conn->spool)) { + write_http_status_line(conn, 304, "Not modified"); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + goto done; + } - /* Prime the connection with some data. */ - const int initial_flush_result = connection_dirserv_flushed_some(conn); - tor_assert_nonfatal(initial_flush_result == 0); + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { + log_debug(LD_DIRSERV, + "Client asked for network status lists, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + geoip_note_ns_response(GEOIP_REJECT_BUSY); goto done; } + tor_addr_t addr; + if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, + &addr, NULL, + time(NULL)); + geoip_note_ns_response(GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (conn->dirreq_id) + geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess, + DIRREQ_DIRECT); + } + + clear_spool = 0; + write_http_response_header(conn, -1, + compression_used, + smartlist_len(conn->spool) == 1 ? lifetime : 0); + if (! compressed) + conn->compress_state = tor_compress_new(0, ZLIB_METHOD, + HIGH_COMPRESSION); + + /* Prime the connection with some data. */ + const int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); + goto done; + done: if (clear_spool) { dir_conn_clear_spool(conn); @@ -3199,7 +3828,7 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const int compressed = args->compression_supported & (1u << ZLIB_METHOD); { int current; ssize_t body_len = 0; @@ -3271,16 +3900,17 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) write_http_status_line(conn, 503, "Directory busy, try again later"); goto vote_done; } - write_http_response_header(conn, body_len ? body_len : -1, compressed, + write_http_response_header(conn, body_len ? body_len : -1, + compressed ? ZLIB_METHOD : NO_METHOD, lifetime); if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(estimated_len)); + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf_zlib(c, strlen(c), conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress(c, strlen(c), conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf(c, strlen(c), TO_CONN(conn))); @@ -3306,7 +3936,7 @@ static int handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const int compressed = args->compression_supported & (1u << ZLIB_METHOD); int clear_spool = 1; { conn->spool = smartlist_new(); @@ -3332,10 +3962,12 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) } clear_spool = 0; - write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + write_http_response_header(conn, -1, + compressed ? ZLIB_METHOD : NO_METHOD, + MICRODESC_CACHE_LIFETIME); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3356,7 +3988,7 @@ static int handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const int compressed = args->compression_supported & (1u << ZLIB_METHOD); const or_options_t *options = get_options(); int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || @@ -3426,9 +4058,11 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) dir_conn_clear_spool(conn); goto done; } - write_http_response_header(conn, -1, compressed, cache_lifetime); + write_http_response_header(conn, -1, + compressed ? ZLIB_METHOD : NO_METHOD, + cache_lifetime); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -3449,7 +4083,7 @@ static int handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const int compressed = args->compression_supported & (1u << ZLIB_METHOD); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new(); @@ -3517,15 +4151,18 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) goto keys_done; } - write_http_response_header(conn, compressed?-1:len, compressed, 60*60); + write_http_response_header(conn, compressed?-1:len, + compressed ? ZLIB_METHOD : NO_METHOD, + 60*60); if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(len)); + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress( + c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf(c->cache_info.signed_descriptor_body, @@ -3556,7 +4193,7 @@ handle_get_hs_descriptor_v2(dir_connection_t *conn, safe_str(escaped(query))); switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { case 1: /* valid */ - write_http_response_header(conn, strlen(descp), 0, 0); + write_http_response_header(conn, strlen(descp), NO_METHOD, 0); connection_write_to_buf(descp, strlen(descp), TO_CONN(conn)); break; case 0: /* well-formed but not present */ @@ -3608,7 +4245,7 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn, } /* Found requested descriptor! Pass it to this nice client. */ - write_http_response_header(conn, strlen(desc_str), 0, 0); + write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0); connection_write_to_buf(desc_str, strlen(desc_str), TO_CONN(conn)); done: @@ -3647,7 +4284,7 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, /* all happy now. send an answer. */ status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); size_t dlen = strlen(status); - write_http_response_header(conn, dlen, 0, 0); + write_http_response_header(conn, dlen, NO_METHOD, 0); connection_write_to_buf(status, dlen, TO_CONN(conn)); tor_free(status); goto done; @@ -3664,7 +4301,7 @@ handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) { const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; size_t len = strlen(robots); - write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME); + write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME); connection_write_to_buf(robots, len, TO_CONN(conn)); } return 0; diff --git a/src/or/directory.h b/src/or/directory.h index 0c5db3e070..125333da37 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -41,27 +41,41 @@ typedef enum { int directory_must_use_begindir(const or_options_t *options); -MOCK_DECL(void, directory_initiate_command_routerstatus, - (const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - struct circuit_guard_state_t *guard_state)); - -void directory_initiate_command_routerstatus_rend(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query, - struct circuit_guard_state_t *guard_state); +/** + * A directory_request_t describes the information about a directory request + * at the client side. It describes what we're going to ask for, which + * directory we're going to ask for it, how we're going to contact that + * directory, and (in some cases) what to do with it when we're done. + */ +typedef struct directory_request_t directory_request_t; +directory_request_t *directory_request_new(uint8_t dir_purpose); +void directory_request_free(directory_request_t *req); +void directory_request_set_or_addr_port(directory_request_t *req, + const tor_addr_port_t *p); +void directory_request_set_dir_addr_port(directory_request_t *req, + const tor_addr_port_t *p); +void directory_request_set_directory_id_digest(directory_request_t *req, + const char *digest); +void directory_request_set_router_purpose(directory_request_t *req, + uint8_t router_purpose); +void directory_request_set_indirection(directory_request_t *req, + dir_indirection_t indirection); +void directory_request_set_resource(directory_request_t *req, + const char *resource); +void directory_request_set_payload(directory_request_t *req, + const char *payload, + size_t payload_len); +void directory_request_set_if_modified_since(directory_request_t *req, + time_t if_modified_since); +void directory_request_set_rend_query(directory_request_t *req, + const rend_data_t *query); + +void directory_request_set_routerstatus(directory_request_t *req, + const routerstatus_t *rs); +void directory_request_add_header(directory_request_t *req, + const char *key, + const char *val); +MOCK_DECL(void, directory_initiate_request, (directory_request_t *request)); int parse_http_response(const char *headers, int *code, time_t *date, compress_method_t *compression, char **response); @@ -72,14 +86,6 @@ int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); int connection_dir_finished_connecting(dir_connection_t *conn); void connection_dir_about_to_close(dir_connection_t *dir_conn); -void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, - const tor_addr_t *dir_addr, uint16_t dir_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); #define DSR_HEX (1<<0) #define DSR_BASE64 (1<<1) @@ -181,7 +187,7 @@ STATIC int handle_post_hs_descriptor(const char *url, const char *body); STATIC char* authdir_type_to_string(dirinfo_type_t auth); STATIC const char * dir_conn_purpose_to_string(int purpose); STATIC int should_use_directory_guards(const or_options_t *options); -STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC compression_level_t choose_compression_level(ssize_t n_bytes); STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, const or_options_t *options); STATIC void find_dl_min_and_max_delay(download_status_t *dls, @@ -192,6 +198,7 @@ STATIC int next_random_exponential_delay(int delay, int max_delay); STATIC int parse_hs_version_from_post(const char *url, const char *prefix, const char **end_pos); +STATIC unsigned parse_accept_encoding_header(const char *h); #endif #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 70b0b22f25..c25dbf8420 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -13,6 +13,7 @@ #include "command.h" #include "connection.h" #include "connection_or.h" +#include "conscache.h" #include "control.h" #include "directory.h" #include "dirserv.h" @@ -570,6 +571,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, !general ? router_purpose_to_string(purpose) : "", !general ? "\n" : "")<0) { *msg = "Couldn't format annotations"; + /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is + * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */ return -1; } @@ -1176,8 +1179,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, + ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; @@ -1211,6 +1214,7 @@ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, const char *flavor_name, const common_digests_t *digests, + const uint8_t *sha3_as_signed, time_t published) { cached_dir_t *new_networkstatus; @@ -1220,6 +1224,8 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); + memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, + DIGEST256_LEN); old_networkstatus = strmap_set(cached_consensuses, flavor_name, new_networkstatus); if (old_networkstatus) @@ -3392,6 +3398,9 @@ spooled_resource_new(dir_spool_source_t source, default: spooled->spool_eagerly = 1; break; + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: + tor_assert_unreached(); + break; } tor_assert(digestlen <= sizeof(spooled->digest)); if (digest) @@ -3399,6 +3408,33 @@ spooled_resource_new(dir_spool_source_t source, return spooled; } +/** + * Create a new spooled_resource_t to spool the contents of <b>entry</b> to + * the user. Return the spooled object on success, or NULL on failure (which + * is probably caused by a failure to map the body of the item from disk). + * + * Adds a reference to entry's reference counter. + */ +spooled_resource_t * +spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry) +{ + spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t)); + spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY; + spooled->spool_eagerly = 0; + consensus_cache_entry_incref(entry); + spooled->consensus_cache_entry = entry; + + int r = consensus_cache_entry_get_body(entry, + &spooled->cce_body, + &spooled->cce_len); + if (r == 0) { + return spooled; + } else { + spooled_resource_free(spooled); + return NULL; + } +} + /** Release all storage held by <b>spooled</b>. */ void spooled_resource_free(spooled_resource_t *spooled) @@ -3410,6 +3446,10 @@ spooled_resource_free(spooled_resource_t *spooled) cached_dir_decref(spooled->cached_dir_ref); } + if (spooled->consensus_cache_entry) { + consensus_cache_entry_decref(spooled->consensus_cache_entry); + } + tor_free(spooled); } @@ -3456,6 +3496,9 @@ spooled_resource_estimate_size(const spooled_resource_t *spooled, return bodylen; } else { cached_dir_t *cached; + if (spooled->consensus_cache_entry) { + return spooled->cce_len; + } if (spooled->cached_dir_ref) { cached = spooled->cached_dir_ref; } else { @@ -3497,15 +3540,16 @@ spooled_resource_flush_some(spooled_resource_t *spooled, /* Absent objects count as "done". */ return SRFS_DONE; } - if (conn->zlib_state) { - connection_write_to_buf_zlib((const char*)body, bodylen, conn, 0); + if (conn->compress_state) { + connection_write_to_buf_compress((const char*)body, bodylen, conn, 0); } else { connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn)); } return SRFS_DONE; } else { cached_dir_t *cached = spooled->cached_dir_ref; - if (cached == NULL) { + consensus_cache_entry_t *cce = spooled->consensus_cache_entry; + if (cached == NULL && cce == NULL) { /* The cached_dir_t hasn't been materialized yet. So let's look it up. */ cached = spooled->cached_dir_ref = spooled_resource_lookup_cached_dir(spooled, NULL); @@ -3517,21 +3561,34 @@ spooled_resource_flush_some(spooled_resource_t *spooled, tor_assert_nonfatal(spooled->cached_dir_offset == 0); } + if (BUG(!cached && !cce)) + return SRFS_DONE; + + int64_t total_len; + const char *ptr; + if (cached) { + total_len = cached->dir_z_len; + ptr = cached->dir_z; + } else { + total_len = spooled->cce_len; + ptr = (const char *)spooled->cce_body; + } /* How many bytes left to flush? */ - int64_t remaining = 0; - remaining = cached->dir_z_len - spooled->cached_dir_offset; + int64_t remaining; + remaining = total_len - spooled->cached_dir_offset; if (BUG(remaining < 0)) return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); - if (conn->zlib_state) { - connection_write_to_buf_zlib(cached->dir_z + spooled->cached_dir_offset, - bytes, conn, 0); + if (conn->compress_state) { + connection_write_to_buf_compress( + ptr + spooled->cached_dir_offset, + bytes, conn, 0); } else { - connection_write_to_buf(cached->dir_z + spooled->cached_dir_offset, + connection_write_to_buf(ptr + spooled->cached_dir_offset, bytes, TO_CONN(conn)); } spooled->cached_dir_offset += bytes; - if (spooled->cached_dir_offset >= (off_t)cached->dir_z_len) { + if (spooled->cached_dir_offset >= (off_t)total_len) { return SRFS_DONE; } else { return SRFS_MORE; @@ -3607,6 +3664,7 @@ spooled_resource_lookup_body(const spooled_resource_t *spooled, return 0; } case DIR_SPOOL_NETWORKSTATUS: + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: default: /* LCOV_EXCL_START */ tor_assert_nonfatal_unreached(); @@ -3788,12 +3846,12 @@ connection_dirserv_flushed_some(dir_connection_t *conn) /* If we get here, we're done. */ smartlist_free(conn->spool); conn->spool = NULL; - if (conn->zlib_state) { - /* Flush the zlib state: there could be more bytes pending in there, and - * we don't want to omit bytes. */ - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; + if (conn->compress_state) { + /* Flush the compression state: there could be more bytes pending in there, + * and we don't want to omit bytes. */ + connection_write_to_buf_compress("", 0, conn, 1); + tor_compress_free(conn->compress_state); + conn->compress_state = NULL; } return 0; } diff --git a/src/or/dirserv.h b/src/or/dirserv.h index f707237ed1..480174d5bb 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -38,6 +38,7 @@ typedef enum dir_spool_source_t { DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, DIR_SPOOL_MICRODESC, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_CONSENSUS_CACHE_ENTRY, } dir_spool_source_t; #define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t) @@ -74,8 +75,15 @@ typedef struct spooled_resource_t { */ struct cached_dir_t *cached_dir_ref; /** - * The current offset into cached_dir. Only used when spool_eagerly is - * false */ + * A different kind of large object that we might be spooling. Also + * reference-counted. Also only used when spool_eagerly is false. + */ + struct consensus_cache_entry_t *consensus_cache_entry; + const uint8_t *cce_body; + size_t cce_len; + /** + * The current offset into cached_dir or cce_body. Only used when + * spool_eagerly is false */ off_t cached_dir_offset; } spooled_resource_t; @@ -110,6 +118,7 @@ cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, const char *flavor_name, const common_digests_t *digests, + const uint8_t *sha3_as_signed, time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key, @@ -184,6 +193,8 @@ int dirserv_read_guardfraction_file(const char *fname, spooled_resource_t *spooled_resource_new(dir_spool_source_t source, const uint8_t *digest, size_t digestlen); +spooled_resource_t *spooled_resource_new_from_cache_entry( + struct consensus_cache_entry_t *entry); void spooled_resource_free(spooled_resource_t *spooled); void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, time_t cutoff, diff --git a/src/or/hs_common.c b/src/or/hs_common.c index 4af3081502..42508126f8 100644 --- a/src/or/hs_common.c +++ b/src/or/hs_common.c @@ -9,6 +9,8 @@ * protocol. **/ +#define HS_COMMON_PRIVATE + #include "or.h" #include "config.h" @@ -50,6 +52,46 @@ hs_check_service_private_dir(const char *username, const char *path, return 0; } +/** Get the default HS time period length in minutes from the consensus. */ +STATIC uint64_t +get_time_period_length(void) +{ + int32_t time_period_length = networkstatus_get_param(NULL, "hsdir-interval", + HS_TIME_PERIOD_LENGTH_DEFAULT, + HS_TIME_PERIOD_LENGTH_MIN, + HS_TIME_PERIOD_LENGTH_MAX); + /* Make sure it's a positive value. */ + tor_assert(time_period_length >= 0); + /* uint64_t will always be able to contain a int32_t */ + return (uint64_t) time_period_length; +} + +/** Get the HS time period number at time <b>now</b> */ +STATIC uint64_t +get_time_period_num(time_t now) +{ + uint64_t time_period_num; + uint64_t time_period_length = get_time_period_length(); + uint64_t minutes_since_epoch = now / 60; + + /* Now subtract half a day to fit the prop224 time period schedule (see + * section [TIME-PERIODS]). */ + tor_assert(minutes_since_epoch > HS_TIME_PERIOD_ROTATION_OFFSET); + minutes_since_epoch -= HS_TIME_PERIOD_ROTATION_OFFSET; + + /* Calculate the time period */ + time_period_num = minutes_since_epoch / time_period_length; + return time_period_num; +} + +/** Get the number of the _upcoming_ HS time period, given that the current + * time is <b>now</b>. */ +uint64_t +hs_get_next_time_period_num(time_t now) +{ + return get_time_period_num(now) + 1; +} + /* Create a new rend_data_t for a specific given <b>version</b>. * Return a pointer to the newly allocated data structure. */ static rend_data_t * diff --git a/src/or/hs_common.h b/src/or/hs_common.h index d7d4228da2..a8fded652a 100644 --- a/src/or/hs_common.h +++ b/src/or/hs_common.h @@ -40,6 +40,15 @@ /* String prefix for the signature of ESTABLISH_INTRO */ #define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1" +/* The default HS time period length */ +#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */ +/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */ +/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */ +/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */ +#define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */ + int hs_check_service_private_dir(const char *username, const char *path, unsigned int dir_group_readable, unsigned int create); @@ -60,5 +69,18 @@ const char *rend_data_get_desc_id(const rend_data_t *rend_data, const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out); +uint64_t hs_get_next_time_period_num(time_t now); + +#ifdef HS_COMMON_PRIVATE + +#ifdef TOR_UNIT_TESTS + +STATIC uint64_t get_time_period_length(void); +STATIC uint64_t get_time_period_num(time_t now); + +#endif /* TOR_UNIT_TESTS */ + +#endif /* HS_COMMON_PRIVATE */ + #endif /* TOR_HS_COMMON_H */ diff --git a/src/or/include.am b/src/or/include.am index 1841bbfe9d..7b031f737b 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -22,6 +22,7 @@ LIBTOR_A_SOURCES = \ src/or/bridges.c \ src/or/buffers.c \ src/or/channel.c \ + src/or/channelpadding.c \ src/or/channeltls.c \ src/or/circpathbias.c \ src/or/circuitbuild.c \ @@ -38,6 +39,7 @@ LIBTOR_A_SOURCES = \ src/or/connection_or.c \ src/or/conscache.c \ src/or/consdiff.c \ + src/or/consdiffmgr.c \ src/or/control.c \ src/or/cpuworker.c \ src/or/dircollate.c \ @@ -119,8 +121,10 @@ src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libev src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ + src/trace/libor-trace.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -132,7 +136,8 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ endif ORHEADERS = \ @@ -140,6 +145,7 @@ ORHEADERS = \ src/or/bridges.h \ src/or/buffers.h \ src/or/channel.h \ + src/or/channelpadding.h \ src/or/channeltls.h \ src/or/circpathbias.h \ src/or/circuitbuild.h \ @@ -156,6 +162,7 @@ ORHEADERS = \ src/or/connection_or.h \ src/or/conscache.h \ src/or/consdiff.h \ + src/or/consdiffmgr.h \ src/or/control.h \ src/or/cpuworker.h \ src/or/dircollate.h \ diff --git a/src/or/main.c b/src/or/main.c index 4505879adc..fe63ddb091 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -54,15 +54,18 @@ #include "buffers.h" #include "channel.h" #include "channeltls.h" +#include "channelpadding.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compress.h" #include "config.h" #include "confparse.h" #include "connection.h" #include "connection_edge.h" #include "connection_or.h" +#include "consdiffmgr.h" #include "control.h" #include "cpuworker.h" #include "crypto_s2k.h" @@ -175,7 +178,7 @@ static int signewnym_is_pending = 0; static unsigned newnym_epoch = 0; /** Smartlist of all open connections. */ -static smartlist_t *connection_array = NULL; +STATIC smartlist_t *connection_array = NULL; /** List of connections that have been marked for close and need to be freed * and removed from connection_array. */ static smartlist_t *closeable_connection_lst = NULL; @@ -1094,8 +1097,9 @@ run_connection_housekeeping(int i, time_t now) } else if (!have_any_circuits && now - or_conn->idle_timeout >= chan->timestamp_last_had_circuits) { - log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " - "[no circuits for %d; timeout %d; %scanonical].", + log_info(LD_OR,"Expiring non-used OR connection "U64_FORMAT" to fd %d " + "(%s:%d) [no circuits for %d; timeout %d; %scanonical].", + U64_PRINTF_ARG(chan->global_identifier), (int)conn->s, conn->address, conn->port, (int)(now - chan->timestamp_last_had_circuits), or_conn->idle_timeout, @@ -1118,6 +1122,8 @@ run_connection_housekeeping(int i, time_t now) memset(&cell,0,sizeof(cell_t)); cell.command = CELL_PADDING; connection_or_write_cell_to_buf(&cell, or_conn); + } else { + channelpadding_decide_to_pad_channel(chan); } } @@ -1184,6 +1190,9 @@ CALLBACK(check_dns_honesty); CALLBACK(write_bridge_ns); CALLBACK(check_fw_helper_app); CALLBACK(heartbeat); +CALLBACK(clean_consdiffmgr); +CALLBACK(reset_padding_counts); +CALLBACK(check_canonical_channels); #undef CALLBACK @@ -1216,6 +1225,9 @@ static periodic_event_item_t periodic_events[] = { CALLBACK(write_bridge_ns), CALLBACK(check_fw_helper_app), CALLBACK(heartbeat), + CALLBACK(clean_consdiffmgr), + CALLBACK(reset_padding_counts), + CALLBACK(check_canonical_channels), END_OF_PERIODIC_EVENTS }; #undef CALLBACK @@ -1471,6 +1483,12 @@ run_scheduled_events(time_t now) /* 11b. check pending unconfigured managed proxies */ if (!net_is_disabled() && pt_proxies_configuration_pending()) pt_configure_remaining_proxies(); + + /* 12. launch diff computations. (This is free if there are none to + * launch.) */ + if (server_mode(options)) { + consdiffmgr_rescan(); + } } /* Periodic callback: rotate the onion keys after the period defined by the @@ -1746,6 +1764,28 @@ write_stats_file_callback(time_t now, const or_options_t *options) return safe_timer_diff(now, next_time_to_write_stats_files); } +#define CHANNEL_CHECK_INTERVAL (60*60) +static int +check_canonical_channels_callback(time_t now, const or_options_t *options) +{ + (void)now; + if (public_server_mode(options)) + channel_check_for_duplicates(); + + return CHANNEL_CHECK_INTERVAL; +} + +static int +reset_padding_counts_callback(time_t now, const or_options_t *options) +{ + if (options->PaddingStatistics) { + rep_hist_prep_published_padding_counts(now); + } + + rep_hist_reset_padding_counts(); + return REPHIST_CELL_PADDING_COUNTS_INTERVAL; +} + /** * Periodic callback: Write bridge statistics to disk if appropriate. */ @@ -2034,6 +2074,17 @@ heartbeat_callback(time_t now, const or_options_t *options) return options->HeartbeatPeriod; } +#define CDM_CLEAN_CALLBACK_INTERVAL 600 +static int +clean_consdiffmgr_callback(time_t now, const or_options_t *options) +{ + (void)now; + if (server_mode(options)) { + consdiffmgr_cleanup(); + } + return CDM_CLEAN_CALLBACK_INTERVAL; +} + /** Timer: used to invoke second_elapsed_callback() once per second. */ static periodic_timer_t *second_timer = NULL; /** Number of libevent errors in the last second: we die if we get too many. */ @@ -2362,6 +2413,8 @@ do_main_loop(void) } handle_signals(1); + monotime_init(); + timers_initialize(); /* load the private keys, if we're supposed to have them, and set up the * TLS context. */ @@ -2998,11 +3051,16 @@ tor_init(int argc, char *argv[]) const char *version = get_version(); log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, " - "OpenSSL %s and Zlib %s.", version, + "OpenSSL %s, Zlib %s, Liblzma %s, and Libzstd %s.", version, get_uname(), tor_libevent_get_version_str(), crypto_openssl_get_version_str(), - tor_zlib_get_version_str()); + tor_compress_supports_method(ZLIB_METHOD) ? + tor_compress_version_str(ZLIB_METHOD) : "N/A", + tor_compress_supports_method(LZMA_METHOD) ? + tor_compress_version_str(LZMA_METHOD) : "N/A", + tor_compress_supports_method(ZSTD_METHOD) ? + tor_compress_version_str(ZSTD_METHOD) : "N/A"); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " @@ -3027,6 +3085,13 @@ tor_init(int argc, char *argv[]) /* The options are now initialised */ const or_options_t *options = get_options(); + /* Initialize channelpadding parameters to defaults until we get + * a consensus */ + channelpadding_new_consensus_params(NULL); + + /* Initialize predicted ports list after loading options */ + predicted_ports_init(); + #ifndef _WIN32 if (geteuid()==0) log_warn(LD_GENERAL,"You are running Tor as root. You don't need to, " @@ -3156,6 +3221,7 @@ tor_free_all(int postfork) sandbox_free_getaddrinfo_cache(); protover_free_all(); bridges_free_all(); + consdiffmgr_free_all(); if (!postfork) { config_free_all(); or_state_free_all(); @@ -3223,6 +3289,9 @@ tor_cleanup(void) rep_hist_record_mtbf_data(now, 0); keypin_close_journal(); } + + timers_shutdown(); + #ifdef USE_DMALLOC dmalloc_log_stats(); #endif @@ -3578,6 +3647,8 @@ sandbox_init_filter(void) OPEN_DATADIR("stats"); STAT_DATADIR("stats"); STAT_DATADIR2("stats", "dirreq-stats"); + + consdiffmgr_register_with_sandbox(&cfg); } init_addrinfo(); @@ -3610,6 +3681,7 @@ tor_main(int argc, char *argv[]) update_approx_time(time(NULL)); tor_threads_init(); + tor_compress_init(); init_logging(0); monotime_init(); #ifdef USE_DMALLOC diff --git a/src/or/main.h b/src/or/main.h index 915d82b7ba..57aa372750 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -92,6 +92,9 @@ STATIC void init_connection_lists(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); +#ifdef TOR_UNIT_TESTS +extern smartlist_t *connection_array; +#endif #endif #endif diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 9d27e576e6..ac5c5c526c 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -46,6 +46,7 @@ #include "config.h" #include "connection.h" #include "connection_or.h" +#include "consdiffmgr.h" #include "control.h" #include "directory.h" #include "dirserv.h" @@ -63,6 +64,7 @@ #include "shared_random.h" #include "transports.h" #include "torcert.h" +#include "channelpadding.h" /** Map from lowercase nickname to identity digest of named server, if any. */ static strmap_t *named_server_map = NULL; @@ -72,11 +74,11 @@ static strmap_t *unnamed_server_map = NULL; /** Most recently received and validated v3 "ns"-flavored consensus network * status. */ -static networkstatus_t *current_ns_consensus = NULL; +STATIC networkstatus_t *current_ns_consensus = NULL; /** Most recently received and validated v3 "microdec"-flavored consensus * network status. */ -static networkstatus_t *current_md_consensus = NULL; +STATIC networkstatus_t *current_md_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ @@ -178,53 +180,74 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } +/** + * Read and and return the cached consensus of type <b>flavorname</b>. If + * <b>unverified</b> is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static char * +networkstatus_read_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char buf[128]; + const char *prefix; + if (unverified_consensus) { + prefix = "unverified"; + } else { + prefix = "cached"; + } + if (flav == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix); + } else { + tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); + } + + char *filename = get_datadir_fname(buf); + char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + tor_free(filename); + return result; +} + +/** Return a new string containing the current cached consensus of flavor + * <b>flavorname</b>. */ +char * +networkstatus_read_cached_consensus(const char *flavorname) + { + int flav = networkstatus_parse_flavor_name(flavorname); + if (flav < 0) + return NULL; + return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); +} + /** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { - char *filename; - char *s; const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; int flav; /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { - char buf[128]; const char *flavor = networkstatus_get_flavor_name(flav); - if (flav == FLAV_NS) { - filename = get_datadir_fname("cached-consensus"); - } else { - tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); - filename = get_datadir_fname(buf); - } - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); if (s) { if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { - log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", - flavor, filename); + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", + flavor); } tor_free(s); } - tor_free(filename); - if (flav == FLAV_NS) { - filename = get_datadir_fname("unverified-consensus"); - } else { - tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); - filename = get_datadir_fname(buf); - } - - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); if (s) { if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS, NULL)) { - log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", - flavor, filename); - } + log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " + "from cache", flavor); + } tor_free(s); } - tor_free(filename); } if (!networkstatus_get_latest_consensus()) { @@ -1966,6 +1989,7 @@ networkstatus_set_current_consensus(const char *consensus, circuit_build_times_new_consensus_params( get_circuit_build_times_mutable(), c); + channelpadding_new_consensus_params(c); } /* Reset the failure count only if this consensus is actually valid. */ @@ -1980,7 +2004,11 @@ networkstatus_set_current_consensus(const char *consensus, dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, + c->digest_sha3_as_signed, c->valid_after); + if (server_mode(get_options())) { + consdiffmgr_add_consensus(consensus, c); + } } if (!from_cache) { diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 8a9f5f0b09..7d148b4ff0 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -16,6 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); +char *networkstatus_read_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free(routerstatus_t *rs); void networkstatus_vote_free(networkstatus_t *ns); @@ -138,7 +139,9 @@ void vote_routerstatus_free(vote_routerstatus_t *rs); #ifdef TOR_UNIT_TESTS STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c, const char *flavor); -#endif // TOR_UNIT_TESTS +extern networkstatus_t *current_ns_consensus; +extern networkstatus_t *current_md_consensus; +#endif #endif #endif diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 9a65fcf014..294fc0df6d 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -159,7 +159,7 @@ onion_skin_TAP_server_handshake( * big. That should be impossible. */ log_info(LD_GENERAL, "crypto_dh_get_public failed."); goto err; - /* LCOV_EXCP_STOP */ + /* LCOV_EXCL_STOP */ } key_material_len = DIGEST_LEN+key_out_len; diff --git a/src/or/or.h b/src/or/or.h index b54de03a1b..9027f312f9 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -71,7 +71,7 @@ #include "tortls.h" #include "torlog.h" #include "container.h" -#include "torgzip.h" +#include "compress.h" #include "address.h" #include "compat_libevent.h" #include "ht.h" @@ -423,12 +423,13 @@ typedef enum { #define DIR_PURPOSE_FETCH_MICRODESC 19 #define DIR_PURPOSE_MAX_ 19 -/** True iff <b>p</b> is a purpose corresponding to uploading data to a - * directory server. */ +/** True iff <b>p</b> is a purpose corresponding to uploading + * data to a directory server. */ #define DIR_PURPOSE_IS_UPLOAD(p) \ ((p)==DIR_PURPOSE_UPLOAD_DIR || \ (p)==DIR_PURPOSE_UPLOAD_VOTE || \ - (p)==DIR_PURPOSE_UPLOAD_SIGNATURES) + (p)==DIR_PURPOSE_UPLOAD_SIGNATURES || \ + (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2) #define EXIT_PURPOSE_MIN_ 1 /** This exit stream wants to do an ordinary connect. */ @@ -895,6 +896,7 @@ typedef enum { #define CELL_RELAY_EARLY 9 #define CELL_CREATE2 10 #define CELL_CREATED2 11 +#define CELL_PADDING_NEGOTIATE 12 #define CELL_VPADDING 128 #define CELL_CERTS 129 @@ -1563,10 +1565,6 @@ typedef struct or_connection_t { * NETINFO cell listed the address we're connected to as recognized. */ unsigned int is_canonical:1; - /** True iff we have decided that the other end of this connection - * is a client. Connections with this flag set should never be used - * to satisfy an EXTEND request. */ - unsigned int is_connection_with_client:1; /** True iff this is an outgoing connection. */ unsigned int is_outgoing:1; unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */ @@ -1773,8 +1771,8 @@ typedef struct dir_connection_t { /** List of spooled_resource_t for objects that we're spooling. We use * it from back to front. */ smartlist_t *spool; - /** The zlib object doing on-the-fly compression for spooled data. */ - tor_zlib_state_t *zlib_state; + /** The compression object doing on-the-fly compression for spooled data. */ + tor_compress_state_t *compress_state; /** What rendezvous service are we querying for? */ rend_data_t *rend_data; @@ -1941,6 +1939,8 @@ typedef struct cached_dir_t { size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ common_digests_t digests; /**< Digests of this object (networkstatus only) */ + /** Sha3 digest (also ns only) */ + uint8_t digest_sha3_as_signed[DIGEST256_LEN]; int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -2641,6 +2641,9 @@ typedef struct networkstatus_t { /** Digests of this document, as signed. */ common_digests_t digests; + /** A SHA3-256 digest of the document, not including signatures: used for + * consensus diffs */ + uint8_t digest_sha3_as_signed[DIGEST256_LEN]; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -3329,6 +3332,13 @@ typedef struct origin_circuit_t { * adjust_exit_policy_from_exitpolicy_failure. */ smartlist_t *prepend_policy; + + /** How long do we wait before closing this circuit if it remains + * completely idle after it was built, in seconds? This value + * is randomized on a per-circuit basis from CircuitsAvailableTimoeut + * to 2*CircuitsAvailableTimoeut. */ + int circuit_idle_timeout; + } origin_circuit_t; struct onion_queue_t; @@ -3470,15 +3480,6 @@ static inline const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT( return DOWNCAST(origin_circuit_t, x); } -/** Bitfield type: things that we're willing to use invalid routers for. */ -typedef enum invalid_router_usage_t { - ALLOW_INVALID_ENTRY =1, - ALLOW_INVALID_EXIT =2, - ALLOW_INVALID_MIDDLE =4, - ALLOW_INVALID_RENDEZVOUS =8, - ALLOW_INVALID_INTRODUCTION=16, -} invalid_router_usage_t; - /* limits for TCP send and recv buffer size used for constrained sockets */ #define MIN_CONSTRAINED_TCP_BUFFER 2048 #define MAX_CONSTRAINED_TCP_BUFFER 262144 /* 256k */ @@ -3604,10 +3605,6 @@ typedef struct { int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our * process for all current and future memory. */ - /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ - smartlist_t *AllowInvalidNodes; - /** Bitmask; derived from AllowInvalidNodes. */ - invalid_router_usage_t AllowInvalid_; config_line_t *ExitPolicy; /**< Lists of exit policy components. */ int ExitPolicyRejectPrivate; /**< Should we not exit to reserved private * addresses, and our own published addresses? @@ -3618,21 +3615,6 @@ typedef struct { * configured ports. */ config_line_t *SocksPolicy; /**< Lists of socks policy components */ config_line_t *DirPolicy; /**< Lists of dir policy components */ - /** Addresses to bind for listening for SOCKS connections. */ - config_line_t *SocksListenAddress; - /** Addresses to bind for listening for transparent pf/netfilter - * connections. */ - config_line_t *TransListenAddress; - /** Addresses to bind for listening for transparent natd connections */ - config_line_t *NATDListenAddress; - /** Addresses to bind for listening for SOCKS connections. */ - config_line_t *DNSListenAddress; - /** Addresses to bind for listening for OR connections. */ - config_line_t *ORListenAddress; - /** Addresses to bind for listening for directory connections. */ - config_line_t *DirListenAddress; - /** Addresses to bind for listening for control connections. */ - config_line_t *ControlListenAddress; /** Local address to bind outbound sockets */ config_line_t *OutboundBindAddress; /** Local address to bind outbound relay sockets */ @@ -3654,7 +3636,6 @@ typedef struct { /** Whether routers accept EXTEND cells to routers with private IPs. */ int ExtendAllowPrivateAddresses; char *User; /**< Name of user to run Tor as. */ - char *Group; /**< Name of group to run Tor as. */ config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */ /** Ports to listen on for extended OR connections. */ config_line_t *ExtORPort_lines; @@ -3757,6 +3738,15 @@ typedef struct { int AvoidDiskWrites; /**< Boolean: should we never cache things to disk? * Not used yet. */ int ClientOnly; /**< Boolean: should we never evolve into a server role? */ + + int ReducedConnectionPadding; /**< Boolean: Should we try to keep connections + open shorter and pad them less against + connection-level traffic analysis? */ + /** Autobool: if auto, then connection padding will be negotiated by client + * and server. If 0, it will be fully disabled. If 1, the client will still + * pad to the server regardless of server support. */ + int ConnectionPadding; + /** To what authority types do we publish our descriptor? Choices are * "v1", "v2", "v3", "bridge", or "". */ smartlist_t *PublishServerDescriptor; @@ -3782,15 +3772,6 @@ typedef struct { /** A routerset that should be used when picking RPs for HS circuits. */ routerset_t *Tor2webRendezvousPoints; - /** Close hidden service client circuits immediately when they reach - * the normal circuit-build timeout, even if they have already sent - * an INTRODUCE1 cell on its way to the service. */ - int CloseHSClientCircuitsImmediatelyOnTimeout; - - /** Close hidden-service-side rendezvous circuits immediately when - * they reach the normal circuit-build timeout. */ - int CloseHSServiceRendCircuitsImmediatelyOnTimeout; - /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct) * circuits between the onion service server, and the introduction and * rendezvous points. (Onion service descriptors are still posted using @@ -3871,8 +3852,8 @@ typedef struct { int CircuitBuildTimeout; /**< Cull non-open circuits that were born at * least this many seconds ago. Used until * adaptive algorithm learns a new value. */ - int CircuitIdleTimeout; /**< Cull open clean circuits that were born - * at least this many seconds ago. */ + int CircuitsAvailableTimeout; /**< Try to have an open circuit for at + least this long after last activity */ int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits * and try a new circuit if the stream has been * waiting for this many seconds. If zero, use @@ -3882,10 +3863,6 @@ typedef struct { * a new one? */ int MaxCircuitDirtiness; /**< Never use circs that were first used more than this interval ago. */ - int PredictedPortsRelevanceTime; /** How long after we've requested a - * connection for a given port, do we want - * to continue to pick exits that support - * that port? */ uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing * to use in a second? */ uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing @@ -3899,8 +3876,6 @@ typedef struct { uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ int NumCPUs; /**< How many CPUs should we try to use? */ -//int RunTesting; /**< If true, create testing circuits to measure how well the -// * other ORs are running. */ config_line_t *RendConfigLines; /**< List of configuration lines * for rendezvous services. */ config_line_t *HidServAuth; /**< List of configuration lines for client-side @@ -4071,8 +4046,6 @@ typedef struct { int NumDirectoryGuards; /**< How many dir guards do we try to establish? * If 0, use value from NumEntryGuards. */ int RephistTrackTime; /**< How many seconds do we keep rephist info? */ - int FastFirstHopPK; /**< If Tor believes it is safe, should we save a third - * of our PK time by sending CREATE_FAST cells? */ /** Should we always fetch our dir info on the mirror schedule (which * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; @@ -4128,16 +4101,6 @@ typedef struct { * if we are a cache). For authorities, this is always true. */ int DownloadExtraInfo; - /** If true, and we are acting as a relay, allow exit circuits even when - * we are the first hop of a circuit. */ - int AllowSingleHopExits; - /** If true, don't allow relays with AllowSingleHopExits=1 to be used in - * circuits that we build. */ - int ExcludeSingleHopRelays; - /** If true, and the controller tells us to use a one-hop circuit, and the - * exit allows it, we use it. */ - int AllowSingleHopCircuits; - /** If true, we convert "www.google.com.foo.exit" addresses on the * socks/trans/natd ports into "www.google.com" addresses that * exit from the node "foo". Disabled by default since attacking @@ -4145,10 +4108,6 @@ typedef struct { * selection. */ int AllowDotExit; - /** If true, we will warn if a user gives us only an IP address - * instead of a hostname. */ - int WarnUnsafeSocks; - /** If true, we're configured to collect statistics on clients * requesting network statuses from us as directory. */ int DirReqStatistics_option; @@ -4165,6 +4124,9 @@ typedef struct { /** If true, the user wants us to collect cell statistics. */ int CellStatistics; + /** If true, the user wants us to collect padding statistics. */ + int PaddingStatistics; + /** If true, the user wants us to collect statistics as entry node. */ int EntryStatistics; @@ -4372,8 +4334,7 @@ typedef struct { int TestingDirAuthVoteGuardIsStrict; /** Relays in a testing network which should be voted HSDir - * regardless of uptime and DirPort. - * Respects VoteOnHidServDirectoriesV2. */ + * regardless of uptime and DirPort. */ routerset_t *TestingDirAuthVoteHSDir; int TestingDirAuthVoteHSDirIsStrict; @@ -4505,8 +4466,6 @@ typedef struct { int IPv6Exit; /**< Do we support exiting to IPv6 addresses? */ - char *TLSECGroup; /**< One of "P256", "P224", or nil for auto */ - /** Fraction: */ double PathsNeededToBuildCircuits; @@ -4815,7 +4774,7 @@ typedef uint32_t build_time_t; double circuit_build_times_quantile_cutoff(void); /** How often in seconds should we build a test circuit */ -#define CBT_DEFAULT_TEST_FREQUENCY 60 +#define CBT_DEFAULT_TEST_FREQUENCY 10 #define CBT_MIN_TEST_FREQUENCY 1 #define CBT_MAX_TEST_FREQUENCY INT32_MAX @@ -5304,7 +5263,8 @@ typedef struct dir_server_t { * address information from published? */ routerstatus_t fake_status; /**< Used when we need to pass this trusted - * dir_server_t to directory_initiate_command_* + * dir_server_t to + * directory_request_set_routerstatus. * as a routerstatus_t. Not updated by the * router-status management code! **/ @@ -5357,7 +5317,6 @@ typedef enum { CRN_NEED_UPTIME = 1<<0, CRN_NEED_CAPACITY = 1<<1, CRN_NEED_GUARD = 1<<2, - CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5, CRN_NEED_DESC = 1<<6, diff --git a/src/or/relay.c b/src/or/relay.c index 5139036327..7082002f84 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -54,6 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -74,6 +75,7 @@ #include "routerlist.h" #include "routerparse.h" #include "scheduler.h" +#include "rephist.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -196,6 +198,82 @@ relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in, return 0; } +/** + * Update channel usage state based on the type of relay cell and + * circuit properties. + * + * This is needed to determine if a client channel is being + * used for application traffic, and if a relay channel is being + * used for multihop circuits and application traffic. The decision + * to pad in channelpadding.c depends upon this info (as well as + * consensus parameters) to decide what channels to pad. + */ +static void +circuit_update_channel_usage(circuit_t *circ, cell_t *cell) +{ + if (CIRCUIT_IS_ORIGIN(circ)) { + /* + * The client state was first set much earlier in + * circuit_send_next_onion_skin(), so we can start padding as early as + * possible. + * + * However, if padding turns out to be expensive, we may want to not do + * it until actual application traffic starts flowing (which is controlled + * via consensus param nf_pad_before_usage). + * + * So: If we're an origin circuit and we've created a full length circuit, + * then any CELL_RELAY cell means application data. Increase the usage + * state of the channel to indicate this. + * + * We want to wait for CELL_RELAY specifically here, so we know that + * the channel was definitely being used for data and not for extends. + * By default, we pad as soon as a channel has been used for *any* + * circuits, so this state is irrelevant to the padding decision in + * the default case. However, if padding turns out to be expensive, + * we would like the ability to avoid padding until we're absolutely + * sure that a channel is used for enough application data to be worth + * padding. + * + * (So it does not matter that CELL_RELAY_EARLY can actually contain + * application data. This is only a load reducing option and that edge + * case does not matter if we're desperately trying to reduce overhead + * anyway. See also consensus parameter nf_pad_before_usage). + */ + if (BUG(!circ->n_chan)) + return; + + if (circ->n_chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS && + cell->command == CELL_RELAY) { + circ->n_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC; + } + } else { + /* If we're a relay circuit, the question is more complicated. Basically: + * we only want to pad connections that carry multihop (anonymous) + * circuits. + * + * We assume we're more than one hop if either the previous hop + * is not a client, or if the previous hop is a client and there's + * a next hop. Then, circuit traffic starts at RELAY_EARLY, and + * user application traffic starts when we see RELAY cells. + */ + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + + if (BUG(!or_circ->p_chan)) + return; + + if (!channel_is_client(or_circ->p_chan) || + (channel_is_client(or_circ->p_chan) && circ->n_chan)) { + if (cell->command == CELL_RELAY_EARLY) { + if (or_circ->p_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS) { + or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS; + } + } else if (cell->command == CELL_RELAY) { + or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC; + } + } + } +} + /** Receive a relay cell: * - Crypt it (encrypt if headed toward the origin or if we <b>are</b> the * origin; decrypt if we're headed toward the exit). @@ -225,10 +303,13 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) { - log_warn(LD_BUG,"relay crypt failed. Dropping connection."); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "relay crypt failed. Dropping connection."); return -END_CIRC_REASON_INTERNAL; } + circuit_update_channel_usage(circ, cell); + if (recognized) { edge_connection_t *conn = NULL; @@ -637,6 +718,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + if (relay_command == RELAY_COMMAND_DROP) + rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) @@ -1528,6 +1612,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, switch (rh.command) { case RELAY_COMMAND_DROP: + rep_hist_padding_count_read(PADDING_TYPE_DROP); // log_info(domain,"Got a relay-level padding cell. Dropping."); return 0; case RELAY_COMMAND_BEGIN: @@ -2453,7 +2538,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); - alloc += tor_zlib_get_total_allocation(); + alloc += tor_compress_get_total_allocation(); const size_t rend_cache_total = rend_cache_get_total_allocation(); alloc += rend_cache_total; if (alloc >= get_options()->MaxMemInQueues_low_threshold) { diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 8d2ae03c9e..9bc2d6289d 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -724,6 +724,9 @@ directory_get_from_hs_dir(const char *desc_id, hs_dir = pick_hsdir(desc_id, desc_id_base32); if (!hs_dir) { /* No suitable hs dir can be found, stop right now. */ + control_event_hs_descriptor_failed(rend_query, NULL, "QUERY_NO_HSDIR"); + control_event_hs_descriptor_content(rend_data_get_address(rend_query), + desc_id_base32, NULL, NULL); return 0; } } @@ -744,6 +747,9 @@ directory_get_from_hs_dir(const char *desc_id, REND_DESC_COOKIE_LEN, 0)<0) { log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); + control_event_hs_descriptor_failed(rend_query, hsdir_fp, "BAD_DESC"); + control_event_hs_descriptor_content(rend_data_get_address(rend_query), + desc_id_base32, hsdir_fp, NULL); return 0; } /* Remove == signs. */ @@ -756,13 +762,15 @@ directory_get_from_hs_dir(const char *desc_id, /* Send fetch request. (Pass query and possibly descriptor cookie so that * they can be written to the directory connection and be referred to when * the response arrives. */ - directory_initiate_command_routerstatus_rend(hs_dir, - DIR_PURPOSE_FETCH_RENDDESC_V2, - ROUTER_PURPOSE_GENERAL, - how_to_fetch, - desc_id_base32, - NULL, 0, 0, - rend_query, NULL); + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_RENDDESC_V2); + directory_request_set_routerstatus(req, hs_dir); + directory_request_set_indirection(req, how_to_fetch); + directory_request_set_resource(req, desc_id_base32); + directory_request_set_rend_query(req, rend_query); + directory_initiate_request(req); + directory_request_free(req); + log_info(LD_REND, "Sending fetch request for v2 descriptor for " "service '%s' with descriptor ID '%s', auth type %d, " "and descriptor cookie '%s' to hidden service " diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 3c4b6775c0..f3b78c4663 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -245,35 +245,23 @@ rend_service_free_all(void) rend_service_list = NULL; } -/** Validate <b>service</b> and add it to <b>service_list</b>, or to - * the global rend_service_list if <b>service_list</b> is NULL. - * Return 0 on success. On failure, free <b>service</b> and return -1. - * Takes ownership of <b>service</b>. - */ +/* Validate a <b>service</b>. Use the <b>service_list</b> to make sure there + * is no duplicate entry for the given service object. Return 0 if valid else + * -1 if not.*/ static int -rend_add_service(smartlist_t *service_list, rend_service_t *service) +rend_validate_service(const smartlist_t *service_list, + const rend_service_t *service) { - int i; - rend_service_port_config_t *p; + int dupe = 0; + tor_assert(service_list); tor_assert(service); - smartlist_t *s_list = rend_get_service_list_mutable(service_list); - /* We must have a service list, even if it's a temporary one, so we can - * check for duplicate services */ - if (BUG(!s_list)) { - return -1; - } - - service->intro_nodes = smartlist_new(); - service->expiring_nodes = smartlist_new(); - if (service->max_streams_per_circuit < 0) { log_warn(LD_CONFIG, "Hidden service (%s) configured with negative max " "streams per circuit.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; + goto invalid; } if (service->max_streams_close_circuit < 0 || @@ -281,87 +269,107 @@ rend_add_service(smartlist_t *service_list, rend_service_t *service) log_warn(LD_CONFIG, "Hidden service (%s) configured with invalid " "max streams handling.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; + goto invalid; } if (service->auth_type != REND_NO_AUTH && - (!service->clients || - smartlist_len(service->clients) == 0)) { - log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but no " - "clients.", + (!service->clients || smartlist_len(service->clients) == 0)) { + log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but " + "no clients.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; + goto invalid; } if (!service->ports || !smartlist_len(service->ports)) { log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.", rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; - } else { - int dupe = 0; - /* XXX This duplicate check has two problems: - * - * a) It's O(n^2), but the same comment from the bottom of - * rend_config_services() should apply. - * - * b) We only compare directory paths as strings, so we can't - * detect two distinct paths that specify the same directory - * (which can arise from symlinks, case-insensitivity, bind - * mounts, etc.). - * - * It also can't detect that two separate Tor instances are trying - * to use the same HiddenServiceDir; for that, we would need a - * lock file. But this is enough to detect a simple mistake that - * at least one person has actually made. - */ - tor_assert(s_list); - if (!rend_service_is_ephemeral(service)) { - /* Skip dupe for ephemeral services. */ - SMARTLIST_FOREACH(s_list, rend_service_t*, ptr, - dupe = dupe || - !strcmp(ptr->directory, service->directory)); - if (dupe) { - log_warn(LD_REND, "Another hidden service is already configured for " - "directory %s.", - rend_service_escaped_dir(service)); - rend_service_free(service); - return -1; - } + goto invalid; + } + + /* XXX This duplicate check has two problems: + * + * a) It's O(n^2), but the same comment from the bottom of + * rend_config_services() should apply. + * + * b) We only compare directory paths as strings, so we can't + * detect two distinct paths that specify the same directory + * (which can arise from symlinks, case-insensitivity, bind + * mounts, etc.). + * + * It also can't detect that two separate Tor instances are trying + * to use the same HiddenServiceDir; for that, we would need a + * lock file. But this is enough to detect a simple mistake that + * at least one person has actually made. + */ + if (!rend_service_is_ephemeral(service)) { + /* Skip dupe for ephemeral services. */ + SMARTLIST_FOREACH(service_list, rend_service_t *, ptr, + dupe = dupe || + !strcmp(ptr->directory, service->directory)); + if (dupe) { + log_warn(LD_REND, "Another hidden service is already configured for " + "directory %s.", + rend_service_escaped_dir(service)); + goto invalid; } - log_debug(LD_REND,"Configuring service with directory %s", - rend_service_escaped_dir(service)); - for (i = 0; i < smartlist_len(service->ports); ++i) { - p = smartlist_get(service->ports, i); - if (!(p->is_unix_addr)) { - log_debug(LD_REND, - "Service maps port %d to %s", - p->virtual_port, - fmt_addrport(&p->real_addr, p->real_port)); - } else { + } + + /* Valid. */ + return 0; + invalid: + return -1; +} + +/** Add it to <b>service_list</b>, or to the global rend_service_list if + * <b>service_list</b> is NULL. Return 0 on success. On failure, free + * <b>service</b> and return -1. Takes ownership of <b>service</b>. */ +static int +rend_add_service(smartlist_t *service_list, rend_service_t *service) +{ + int i; + rend_service_port_config_t *p; + + tor_assert(service); + + smartlist_t *s_list = rend_get_service_list_mutable(service_list); + /* We must have a service list, even if it's a temporary one, so we can + * check for duplicate services */ + if (BUG(!s_list)) { + return -1; + } + + service->intro_nodes = smartlist_new(); + service->expiring_nodes = smartlist_new(); + + log_debug(LD_REND,"Configuring service with directory %s", + rend_service_escaped_dir(service)); + for (i = 0; i < smartlist_len(service->ports); ++i) { + p = smartlist_get(service->ports, i); + if (!(p->is_unix_addr)) { + log_debug(LD_REND, + "Service maps port %d to %s", + p->virtual_port, + fmt_addrport(&p->real_addr, p->real_port)); + } else { #ifdef HAVE_SYS_UN_H - log_debug(LD_REND, - "Service maps port %d to socket at \"%s\"", - p->virtual_port, p->unix_addr); + log_debug(LD_REND, + "Service maps port %d to socket at \"%s\"", + p->virtual_port, p->unix_addr); #else - log_warn(LD_BUG, - "Service maps port %d to an AF_UNIX socket, but we " - "have no AF_UNIX support on this platform. This is " - "probably a bug.", - p->virtual_port); - rend_service_free(service); - return -1; + log_warn(LD_BUG, + "Service maps port %d to an AF_UNIX socket, but we " + "have no AF_UNIX support on this platform. This is " + "probably a bug.", + p->virtual_port); + rend_service_free(service); + return -1; #endif /* defined(HAVE_SYS_UN_H) */ - } } - /* The service passed all the checks */ - tor_assert(s_list); - smartlist_add(s_list, service); - return 0; } - /* NOTREACHED */ + /* The service passed all the checks */ + tor_assert(s_list); + smartlist_add(s_list, service); + return 0; } /** Return a new rend_service_port_config_t with its path set to @@ -671,13 +679,19 @@ rend_config_services(const or_options_t *options, int validate_only) for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - /* register the service we just finished parsing - * this code registers every service except the last one parsed, - * which is registered below the loop */ - if (rend_service_check_dir_and_add(rend_service_staging_list, options, - service, validate_only) < 0) { - service = NULL; - goto free_and_return; + if (service) { + /* Validate and register the service we just finished parsing this + * code registers every service except the last one parsed, which is + * validated and registered below the loop. */ + if (rend_validate_service(rend_service_staging_list, service) < 0) { + goto free_and_return; + } + if (rend_service_check_dir_and_add(rend_service_staging_list, options, + service, validate_only) < 0) { + /* The above frees the service on error so nullify the pointer. */ + service = NULL; + goto free_and_return; + } } service = tor_malloc_zero(sizeof(rend_service_t)); service->directory = tor_strdup(line->value); @@ -872,14 +886,23 @@ rend_config_services(const or_options_t *options, int validate_only) } } } + /* Validate the last service that we just parsed. */ + if (service && + rend_validate_service(rend_service_staging_list, service) < 0) { + goto free_and_return; + } /* register the final service after we have finished parsing all services * this code only registers the last service, other services are registered * within the loop. It is ok for this service to be NULL, it is ignored. */ if (rend_service_check_dir_and_add(rend_service_staging_list, options, service, validate_only) < 0) { + /* Service object is freed on error so nullify pointer. */ service = NULL; goto free_and_return; } + /* The service is in the staging list so nullify pointer to avoid double + * free of this object in case of error because we lost ownership of it at + * this point. */ service = NULL; /* Free the newly added services if validating */ @@ -1054,20 +1077,13 @@ rend_log_intro_limit(const rend_service_t *service, int min_severity) } time_t intro_period_elapsed = time(NULL) - service->intro_period_started; tor_assert_nonfatal(intro_period_elapsed >= 0); - /* We delayed resuming circuits longer than expected */ - int exceeded_elapsed = (intro_period_elapsed > INTRO_CIRC_RETRY_PERIOD + - INTRO_CIRC_RETRY_PERIOD_SLOP); - if (exceeded_elapsed) { - severity = LOG_WARN; - } log_fn(severity, LD_REND, "Hidden service %s %s %d intro points in the last " - "%d seconds%s. Intro circuit launches are limited to %d per %d " + "%d seconds. Intro circuit launches are limited to %d per %d " "seconds.", service->service_id, exceeded_limit ? "exceeded launch limit with" : "launched", service->n_intro_circuits_launched, (int)intro_period_elapsed, - exceeded_elapsed ? " (delayed)" : "", rend_max_intro_circs_per_period(service->n_intro_points_wanted), INTRO_CIRC_RETRY_PERIOD); rend_service_dump_stats(severity); @@ -3712,13 +3728,16 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, * request. Lookup is made in rend_service_desc_has_uploaded(). */ rend_data = rend_data_client_create(service_id, desc->desc_id, NULL, REND_NO_AUTH); - directory_initiate_command_routerstatus_rend(hs_dir, - DIR_PURPOSE_UPLOAD_RENDDESC_V2, - ROUTER_PURPOSE_GENERAL, - DIRIND_ANONYMOUS, NULL, - desc->desc_str, - strlen(desc->desc_str), - 0, rend_data, NULL); + directory_request_t *req = + directory_request_new(DIR_PURPOSE_UPLOAD_RENDDESC_V2); + directory_request_set_routerstatus(req, hs_dir); + directory_request_set_indirection(req, DIRIND_ANONYMOUS); + directory_request_set_payload(req, + desc->desc_str, strlen(desc->desc_str)); + directory_request_set_rend_query(req, rend_data); + directory_initiate_request(req); + directory_request_free(req); + rend_data_free(rend_data); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); @@ -4186,8 +4205,6 @@ rend_consider_services_intro_points(void) const node_t *node; rend_intro_point_t *intro; router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC; - if (get_options()->AllowInvalid_ & ALLOW_INVALID_INTRODUCTION) - flags |= CRN_ALLOW_INVALID; router_crn_flags_t direct_flags = flags; direct_flags |= CRN_PREF_ADDR; direct_flags |= CRN_DIRECT_CONN; diff --git a/src/or/rephist.c b/src/or/rephist.c index 231130f13c..72a5cc5a9b 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -84,9 +84,13 @@ #include "router.h" #include "routerlist.h" #include "ht.h" +#include "channelpadding.h" + +#include "channelpadding.h" +#include "connection_or.h" static void bw_arrays_init(void); -static void predicted_ports_init(void); +static void predicted_ports_alloc(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -165,6 +169,44 @@ typedef struct or_history_t { digestmap_t *link_history_map; } or_history_t; +/** + * This structure holds accounting needed to calculate the padding overhead. + */ +typedef struct padding_counts_t { + /** Total number of cells we have received, including padding */ + uint64_t read_cell_count; + /** Total number of cells we have sent, including padding */ + uint64_t write_cell_count; + /** Total number of CELL_PADDING cells we have received */ + uint64_t read_pad_cell_count; + /** Total number of CELL_PADDING cells we have sent */ + uint64_t write_pad_cell_count; + /** Total number of read cells on padding-enabled conns */ + uint64_t enabled_read_cell_count; + /** Total number of sent cells on padding-enabled conns */ + uint64_t enabled_write_cell_count; + /** Total number of read CELL_PADDING cells on padding-enabled cons */ + uint64_t enabled_read_pad_cell_count; + /** Total number of sent CELL_PADDING cells on padding-enabled cons */ + uint64_t enabled_write_pad_cell_count; + /** Total number of RELAY_DROP cells we have received */ + uint64_t read_drop_cell_count; + /** Total number of RELAY_DROP cells we have sent */ + uint64_t write_drop_cell_count; + /** The maximum number of padding timers we've seen in 24 hours */ + uint64_t maximum_chanpad_timers; + /** When did we first copy padding_current into padding_published? */ + char first_published_at[ISO_TIME_LEN+1]; +} padding_counts_t; + +/** Holds the current values of our padding statistics. + * It is not published until it is transferred to padding_published. */ +static padding_counts_t padding_current; + +/** Remains fixed for a 24 hour period, and then is replaced + * by a redacted copy of padding_current */ +static padding_counts_t padding_published; + /** When did we last multiply all routers' weighted_run_length and * total_run_weights by STABILITY_ALPHA? */ static time_t stability_last_downrated = 0; @@ -264,7 +306,7 @@ rep_hist_init(void) { history_map = digestmap_new(); bw_arrays_init(); - predicted_ports_init(); + predicted_ports_alloc(); } /** Helper: note that we are no longer connected to the router with history @@ -1758,6 +1800,40 @@ typedef struct predicted_port_t { /** A list of port numbers that have been used recently. */ static smartlist_t *predicted_ports_list=NULL; +/** How long do we keep predicting circuits? */ +static int prediction_timeout=0; +/** When was the last time we added a prediction entry (HS or port) */ +static time_t last_prediction_add_time=0; + +/** + * How much time left until we stop predicting circuits? + */ +int +predicted_ports_prediction_time_remaining(time_t now) +{ + time_t idle_delta = now - last_prediction_add_time; + + /* Protect against overflow of return value. This can happen if the clock + * jumps backwards in time. Update the last prediction time (aka last + * active time) to prevent it. This update is preferable to using monotonic + * time because it prevents clock jumps into the past from simply causing + * very long idle timeouts while the monotonic time stands still. */ + if (last_prediction_add_time > now) { + last_prediction_add_time = now; + idle_delta = 0; + } + + /* Protect against underflow of the return value. This can happen for very + * large periods of inactivity/system sleep. */ + if (idle_delta > prediction_timeout) + return 0; + + if (BUG((prediction_timeout - idle_delta) > INT_MAX)) { + return INT_MAX; + } + + return (int)(prediction_timeout - idle_delta); +} /** We just got an application request for a connection with * port <b>port</b>. Remember it for the future, so we can keep @@ -1767,21 +1843,40 @@ static void add_predicted_port(time_t now, uint16_t port) { predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t)); + + // If the list is empty, re-randomize predicted ports lifetime + if (!any_predicted_circuits(now)) { + prediction_timeout = channelpadding_get_circuits_available_timeout(); + } + + last_prediction_add_time = now; + + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ building " + "for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); + pp->port = port; pp->time = now; rephist_total_alloc += sizeof(*pp); smartlist_add(predicted_ports_list, pp); } -/** Initialize whatever memory and structs are needed for predicting +/** + * Allocate whatever memory and structs are needed for predicting * which ports will be used. Also seed it with port 80, so we'll build * circuits on start-up. */ static void -predicted_ports_init(void) +predicted_ports_alloc(void) { predicted_ports_list = smartlist_new(); - add_predicted_port(time(NULL), 80); /* add one to kickstart us */ +} + +void +predicted_ports_init(void) +{ + add_predicted_port(time(NULL), 443); // Add a port to get us started } /** Free whatever memory is needed for predicting which ports will @@ -1812,6 +1907,12 @@ rep_hist_note_used_port(time_t now, uint16_t port) SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { if (pp->port == port) { pp->time = now; + + last_prediction_add_time = now; + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ " + "building for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); return; } } SMARTLIST_FOREACH_END(pp); @@ -1828,7 +1929,8 @@ rep_hist_get_predicted_ports(time_t now) int predicted_circs_relevance_time; smartlist_t *out = smartlist_new(); tor_assert(predicted_ports_list); - predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + + predicted_circs_relevance_time = prediction_timeout; /* clean out obsolete entries */ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { @@ -1888,6 +1990,18 @@ static time_t predicted_internal_capacity_time = 0; void rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity) { + // If the list is empty, re-randomize predicted ports lifetime + if (!any_predicted_circuits(now)) { + prediction_timeout = channelpadding_get_circuits_available_timeout(); + } + + last_prediction_add_time = now; + + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ building " + "for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); + predicted_internal_time = now; if (need_uptime) predicted_internal_uptime_time = now; @@ -1901,7 +2015,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, int *need_capacity) { int predicted_circs_relevance_time; - predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + + predicted_circs_relevance_time = prediction_timeout; if (!predicted_internal_time) { /* initialize it */ predicted_internal_time = now; @@ -1923,7 +2038,7 @@ int any_predicted_circuits(time_t now) { int predicted_circs_relevance_time; - predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + predicted_circs_relevance_time = prediction_timeout; return smartlist_len(predicted_ports_list) || predicted_internal_time + predicted_circs_relevance_time >= now; @@ -3210,8 +3325,7 @@ rep_hist_hs_stats_write(time_t now) return start_of_hs_stats_interval + WRITE_STATS_INTERVAL; } -#define MAX_LINK_PROTO_TO_LOG 4 -static uint64_t link_proto_count[MAX_LINK_PROTO_TO_LOG+1][2]; +static uint64_t link_proto_count[MAX_LINK_PROTO+1][2]; /** Note that we negotiated link protocol version <b>link_proto</b>, on * a connection that started here iff <b>started_here</b> is true. @@ -3220,7 +3334,7 @@ void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here) { started_here = !!started_here; /* force to 0 or 1 */ - if (link_proto > MAX_LINK_PROTO_TO_LOG) { + if (link_proto > MAX_LINK_PROTO) { log_warn(LD_BUG, "Can't log link protocol %u", link_proto); return; } @@ -3228,6 +3342,165 @@ rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here) link_proto_count[link_proto][started_here]++; } +/** + * Update the maximum count of total pending channel padding timers + * in this period. + */ +void +rep_hist_padding_count_timers(uint64_t num_timers) +{ + if (num_timers > padding_current.maximum_chanpad_timers) { + padding_current.maximum_chanpad_timers = num_timers; + } +} + +/** + * Count a cell that we sent for padding overhead statistics. + * + * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be + * counted for PADDING_TYPE_TOTAL. + */ +void +rep_hist_padding_count_write(padding_type_t type) +{ + switch (type) { + case PADDING_TYPE_DROP: + padding_current.write_drop_cell_count++; + break; + case PADDING_TYPE_CELL: + padding_current.write_pad_cell_count++; + break; + case PADDING_TYPE_TOTAL: + padding_current.write_cell_count++; + break; + case PADDING_TYPE_ENABLED_TOTAL: + padding_current.enabled_write_cell_count++; + break; + case PADDING_TYPE_ENABLED_CELL: + padding_current.enabled_write_pad_cell_count++; + break; + } +} + +/** + * Count a cell that we've received for padding overhead statistics. + * + * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be + * counted for PADDING_TYPE_TOTAL. + */ +void +rep_hist_padding_count_read(padding_type_t type) +{ + switch (type) { + case PADDING_TYPE_DROP: + padding_current.read_drop_cell_count++; + break; + case PADDING_TYPE_CELL: + padding_current.read_pad_cell_count++; + break; + case PADDING_TYPE_TOTAL: + padding_current.read_cell_count++; + break; + case PADDING_TYPE_ENABLED_TOTAL: + padding_current.enabled_read_cell_count++; + break; + case PADDING_TYPE_ENABLED_CELL: + padding_current.enabled_read_pad_cell_count++; + break; + } +} + +/** + * Reset our current padding statistics. Called once every 24 hours. + */ +void +rep_hist_reset_padding_counts(void) +{ + memset(&padding_current, 0, sizeof(padding_current)); +} + +/** + * Copy our current cell counts into a structure for listing in our + * extra-info descriptor. Also perform appropriate rounding and redaction. + * + * This function is called once every 24 hours. + */ +#define MIN_CELL_COUNTS_TO_PUBLISH 1 +#define ROUND_CELL_COUNTS_TO 10000 +void +rep_hist_prep_published_padding_counts(time_t now) +{ + memcpy(&padding_published, &padding_current, sizeof(padding_published)); + + if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH || + padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) { + memset(&padding_published, 0, sizeof(padding_published)); + return; + } + + format_iso_time(padding_published.first_published_at, now); +#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \ + ROUND_CELL_COUNTS_TO) + ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_cell_count); + ROUND_AND_SET_COUNT(padding_published.read_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count); +#undef ROUND_AND_SET_COUNT +} + +/** + * Returns an allocated string for extra-info documents for publishing + * padding statistics from the last 24 hour interval. + */ +char * +rep_hist_get_padding_count_lines(void) +{ + char *result = NULL; + + if (!padding_published.read_cell_count || + !padding_published.write_cell_count) { + return NULL; + } + + tor_asprintf(&result, "padding-counts %s (%d s)" + " bin-size="U64_FORMAT + " write-drop="U64_FORMAT + " write-pad="U64_FORMAT + " write-total="U64_FORMAT + " read-drop="U64_FORMAT + " read-pad="U64_FORMAT + " read-total="U64_FORMAT + " enabled-read-pad="U64_FORMAT + " enabled-read-total="U64_FORMAT + " enabled-write-pad="U64_FORMAT + " enabled-write-total="U64_FORMAT + " max-chanpad-timers="U64_FORMAT + "\n", + padding_published.first_published_at, + REPHIST_CELL_PADDING_COUNTS_INTERVAL, + U64_PRINTF_ARG(ROUND_CELL_COUNTS_TO), + U64_PRINTF_ARG(padding_published.write_drop_cell_count), + U64_PRINTF_ARG(padding_published.write_pad_cell_count), + U64_PRINTF_ARG(padding_published.write_cell_count), + U64_PRINTF_ARG(padding_published.read_drop_cell_count), + U64_PRINTF_ARG(padding_published.read_pad_cell_count), + U64_PRINTF_ARG(padding_published.read_cell_count), + U64_PRINTF_ARG(padding_published.enabled_read_pad_cell_count), + U64_PRINTF_ARG(padding_published.enabled_read_cell_count), + U64_PRINTF_ARG(padding_published.enabled_write_pad_cell_count), + U64_PRINTF_ARG(padding_published.enabled_write_cell_count), + U64_PRINTF_ARG(padding_published.maximum_chanpad_timers) + ); + + return result; +} + /** Log a heartbeat message explaining how many connections of each link * protocol version we have used. */ diff --git a/src/or/rephist.h b/src/or/rephist.h index 6dd88a3544..2b1c2e7ec7 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -48,6 +48,7 @@ double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); long rep_hist_get_weighted_time_known(const char *id, time_t when); int rep_hist_have_measured_enough_stability(void); +void predicted_ports_init(void); void rep_hist_note_used_port(time_t now, uint16_t port); smartlist_t *rep_hist_get_predicted_ports(time_t now); void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports); @@ -59,6 +60,7 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime, int any_predicted_circuits(time_t now); int rep_hist_circbuilding_dormant(time_t now); +int predicted_ports_prediction_time_remaining(time_t now); void note_crypto_pk_op(pk_op_t operation); void dump_pk_ops(int severity); @@ -119,5 +121,30 @@ extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; #endif +/** + * Represents the type of a cell for padding accounting + */ +typedef enum padding_type_t { + /** A RELAY_DROP cell */ + PADDING_TYPE_DROP, + /** A CELL_PADDING cell */ + PADDING_TYPE_CELL, + /** Total counts of padding and non-padding together */ + PADDING_TYPE_TOTAL, + /** Total cell counts for all padding-enabled channels */ + PADDING_TYPE_ENABLED_TOTAL, + /** CELL_PADDING counts for all padding-enabled channels */ + PADDING_TYPE_ENABLED_CELL +} padding_type_t; + +/** The amount of time over which the padding cell counts were counted */ +#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60) +void rep_hist_padding_count_read(padding_type_t type); +void rep_hist_padding_count_write(padding_type_t type); +char *rep_hist_get_padding_count_lines(void); +void rep_hist_reset_padding_counts(void); +void rep_hist_prep_published_padding_counts(time_t now); +void rep_hist_padding_count_timers(uint64_t num_timers); + #endif diff --git a/src/or/router.c b/src/or/router.c index c02cf5321e..642f415a38 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -779,12 +779,6 @@ router_initialize_tls_context(void) int lifetime = options->SSLKeyLifetime; if (public_server_mode(options)) flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER; - if (options->TLSECGroup) { - if (!strcasecmp(options->TLSECGroup, "P256")) - flags |= TOR_TLS_CTX_USE_ECDHE_P256; - else if (!strcasecmp(options->TLSECGroup, "P224")) - flags |= TOR_TLS_CTX_USE_ECDHE_P224; - } if (!lifetime) { /* we should guess a good ssl cert lifetime */ /* choose between 5 and 365 days, and round to the day */ @@ -1470,13 +1464,23 @@ consider_testing_reachability(int test_or, int test_dir) !connection_get_by_type_addr_port_purpose( CONN_TYPE_DIR, &addr, me->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)) { + tor_addr_port_t my_orport, my_dirport; + memcpy(&my_orport.addr, &addr, sizeof(addr)); + memcpy(&my_dirport.addr, &addr, sizeof(addr)); + my_orport.port = me->or_port; + my_dirport.port = me->dir_port; /* ask myself, via tor, for my server descriptor. */ - directory_initiate_command(&addr, me->or_port, - &addr, me->dir_port, - me->cache_info.identity_digest, - DIR_PURPOSE_FETCH_SERVERDESC, - ROUTER_PURPOSE_GENERAL, - DIRIND_ANON_DIRPORT, "authority.z", NULL, 0, 0); + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); + directory_request_set_or_addr_port(req, &my_orport); + directory_request_set_dir_addr_port(req, &my_dirport); + directory_request_set_directory_id_digest(req, + me->cache_info.identity_digest); + // ask via an anon circuit, connecting to our dirport. + directory_request_set_indirection(req, DIRIND_ANON_DIRPORT); + directory_request_set_resource(req, "authority.z"); + directory_initiate_request(req); + directory_request_free(req); } } @@ -1653,8 +1657,7 @@ MOCK_IMPL(int, server_mode,(const or_options_t *options)) { if (options->ClientOnly) return 0; - /* XXXX I believe we can kill off ORListenAddress here.*/ - return (options->ORPort_set || options->ORListenAddress); + return (options->ORPort_set); } /** Return true iff we are trying to be a non-bridge server. @@ -2928,7 +2931,7 @@ router_dump_router_to_string(routerinfo_t *router, "onion-key\n%s" "signing-key\n%s" "%s%s" - "%s%s%s%s", + "%s%s%s", router->nickname, address, router->or_port, @@ -2951,8 +2954,7 @@ router_dump_router_to_string(routerinfo_t *router, ntor_cc_line ? ntor_cc_line : "", family_line, we_are_hibernating() ? "hibernating 1\n" : "", - "hidden-service-dir\n", - options->AllowSingleHopExits ? "allow-single-hop-exits\n" : ""); + "hidden-service-dir\n"); if (options->ContactInfo && strlen(options->ContactInfo)) { const char *ci = options->ContactInfo; @@ -3281,6 +3283,12 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, } } + if (options->PaddingStatistics) { + contents = rep_hist_get_padding_count_lines(); + if (contents) + smartlist_add(chunks, contents); + } + /* Add information about the pluggable transports we support. */ if (options->ServerTransportPlugin) { char *pluggable_transports = pt_get_extra_info_descriptor_string(); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 0b0bb4b1d2..cba0d9200d 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -947,6 +947,7 @@ authority_certs_fetch_resource_impl(const char *resource, const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; + directory_request_t *req = NULL; /* If we've just downloaded a consensus from a bridge, re-use that * bridge */ if (options->UseBridges && node && node->ri && !get_via_tor) { @@ -955,23 +956,25 @@ authority_certs_fetch_resource_impl(const char *resource, /* we are willing to use a non-preferred address if we need to */ fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &or_ap); - directory_initiate_command(&or_ap.addr, or_ap.port, - NULL, 0, /*no dirport*/ - dir_hint, - DIR_PURPOSE_FETCH_CERTIFICATE, - 0, - indirection, - resource, NULL, 0, 0); - return; - } - if (rs) { - /* If we've just downloaded a consensus from a directory, re-use that + req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); + directory_request_set_or_addr_port(req, &or_ap); + if (dir_hint) + directory_request_set_directory_id_digest(req, dir_hint); + } else if (rs) { + /* And if we've just downloaded a consensus from a directory, re-use that * directory */ - directory_initiate_command_routerstatus(rs, - DIR_PURPOSE_FETCH_CERTIFICATE, - 0, indirection, resource, NULL, - 0, 0, NULL); + req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); + directory_request_set_routerstatus(req, rs); + } + + if (req) { + /* We've set up a request object -- fill in the other request fields, and + * send the request. */ + directory_request_set_indirection(req, indirection); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); return; } @@ -2317,17 +2320,16 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router) * we can pick a node for a circuit. */ void -router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, - int need_uptime, int need_capacity, - int need_guard, int need_desc, - int pref_addr, int direct_conn) +router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime, + int need_capacity, int need_guard, + int need_desc, int pref_addr, + int direct_conn) { const int check_reach = !router_skip_or_reachability(get_options(), pref_addr); /* XXXX MOVE */ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { - if (!node->is_running || - (!node->is_valid && !allow_invalid)) + if (!node->is_running || !node->is_valid) continue; if (need_desc && !(node->ri || (node->rs && node->md))) continue; @@ -2773,8 +2775,6 @@ node_sl_choose_by_bandwidth(const smartlist_t *sl, * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the * advertised capacity of each router. - * If <b>CRN_ALLOW_INVALID</b> is not set in flags, consider only Valid - * routers. * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers. * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if * picking an exit node, otherwise we weight bandwidths for picking a relay @@ -2795,7 +2795,6 @@ router_choose_random_node(smartlist_t *excludedsmartlist, const int need_uptime = (flags & CRN_NEED_UPTIME) != 0; const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; - const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; const int need_desc = (flags & CRN_NEED_DESC) != 0; const int pref_addr = (flags & CRN_PREF_ADDR) != 0; @@ -2811,20 +2810,17 @@ router_choose_random_node(smartlist_t *excludedsmartlist, rule = weight_for_exit ? WEIGHT_FOR_EXIT : (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); - /* Exclude relays that allow single hop exit circuits, if the user - * wants to (such relays might be risky) */ - if (get_options()->ExcludeSingleHopRelays) { - SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node, - if (node_allows_single_hop_exits(node)) { - smartlist_add(excludednodes, node); - }); - } + /* Exclude relays that allow single hop exit circuits. This is an obsolete + * option since 0.2.9.2-alpha and done by default in 0.3.1.0-alpha. */ + SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node, + if (node_allows_single_hop_exits(node)) { + smartlist_add(excludednodes, node); + }); if ((r = routerlist_find_my_routerinfo())) routerlist_add_node_and_family(excludednodes, r); - router_add_running_nodes_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, + router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity, need_guard, need_desc, pref_addr, direct_conn); log_debug(LD_CIRC, @@ -3045,8 +3041,8 @@ router_get_by_extrainfo_digest,(const char *digest)) /** Return the signed descriptor for the extrainfo_t in our routerlist whose * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info * document is known. */ -signed_descriptor_t * -extrainfo_get_by_descriptor_digest(const char *digest) +MOCK_IMPL(signed_descriptor_t *, +extrainfo_get_by_descriptor_digest,(const char *digest)) { extrainfo_t *ei; tor_assert(digest); @@ -4932,10 +4928,11 @@ MOCK_IMPL(STATIC void, initiate_descriptor_downloads, if (source) { /* We know which authority or directory mirror we want. */ - directory_initiate_command_routerstatus(source, purpose, - ROUTER_PURPOSE_GENERAL, - DIRIND_ONEHOP, - resource, NULL, 0, 0, NULL); + directory_request_t *req = directory_request_new(purpose); + directory_request_set_routerstatus(req, source); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); } else { directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource, pds_flags, DL_WANT_ANY_DIRSERVER); diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 5376369e8d..25b2f64350 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -62,10 +62,10 @@ int router_skip_or_reachability(const or_options_t *options, int try_ip_pref); int router_get_my_share_of_directory_requests(double *v3_share_out); void router_reset_status_download_failures(void); int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2); -void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, - int need_uptime, int need_capacity, - int need_guard, int need_desc, - int pref_addr, int direct_conn); +void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime, + int need_capacity, int need_guard, + int need_desc, int pref_addr, + int direct_conn); const routerinfo_t *routerlist_find_my_routerinfo(void); uint32_t router_get_advertised_bandwidth(const routerinfo_t *router); @@ -92,7 +92,8 @@ routerinfo_t *router_get_mutable_by_digest(const char *digest); signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest, (const char *digest)); -signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); +MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest, + (const char *digest)); const char *signed_descriptor_get_body(const signed_descriptor_t *desc); const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc); routerlist_t *router_get_routerlist(void); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index fc0a4ab50a..fa79cf7132 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -359,6 +359,7 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, const char *end_str, char end_c, + int log_severity, const char **start_out, const char **end_out); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, @@ -988,6 +989,41 @@ router_get_router_hash(const char *s, size_t s_len, char *digest) DIGEST_SHA1); } +/** Try to find the start and end of the signed portion of a networkstatus + * document in <b>s</b>. On success, set <b>start_out</b> to the first + * character of the document, and <b>end_out</b> to a position one after the + * final character of the signed document, and return 0. On failure, return + * -1. */ +int +router_get_networkstatus_v3_signed_boundaries(const char *s, + const char **start_out, + const char **end_out) +{ + return router_get_hash_impl_helper(s, strlen(s), + "network-status-version", + "\ndirectory-signature", + ' ', LOG_INFO, + start_out, end_out); +} + +/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the + * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no + * signed portion can be identified. Return 0 on success, -1 on failure. */ +int +router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, + const char *s) +{ + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) { + start = s; + end = s + strlen(s); + } + tor_assert(start); + tor_assert(end); + return crypto_digest256((char*)digest_out, start, end-start, + DIGEST_SHA3_256); +} + /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int @@ -1787,7 +1823,8 @@ router_parse_entry_from_string(const char *s, const char *end, if (router_get_hash_impl_helper(s, end-s, "router ", "\nrouter-sig-ed25519", - ' ', &signed_start, &signed_end) < 0) { + ' ', LOG_WARN, + &signed_start, &signed_end) < 0) { log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor"); goto err; } @@ -2140,7 +2177,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (router_get_hash_impl_helper(s, end-s, "extra-info ", "\nrouter-sig-ed25519", - ' ', &signed_start, &signed_end) < 0) { + ' ', LOG_WARN, + &signed_start, &signed_end) < 0) { log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo"); goto err; } @@ -3346,6 +3384,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; common_digests_t ns_digests; + uint8_t sha3_as_signed[DIGEST256_LEN]; const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; struct in_addr in; @@ -3359,7 +3398,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests) || + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -3376,6 +3416,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns = tor_malloc_zero(sizeof(networkstatus_t)); memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed)); tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); tor_assert(tok); @@ -4471,16 +4512,18 @@ static int router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, const char *end_str, char end_c, + int log_severity, const char **start_out, const char **end_out) { const char *start, *end; start = tor_memstr(s, s_len, start_str); if (!start) { - log_warn(LD_DIR,"couldn't find start of hashed material \"%s\"",start_str); + log_fn(log_severity,LD_DIR, + "couldn't find start of hashed material \"%s\"",start_str); return -1; } if (start != s && *(start-1) != '\n') { - log_warn(LD_DIR, + log_fn(log_severity,LD_DIR, "first occurrence of \"%s\" is not at the start of a line", start_str); return -1; @@ -4488,12 +4531,14 @@ router_get_hash_impl_helper(const char *s, size_t s_len, end = tor_memstr(start+strlen(start_str), s_len - (start-s) - strlen(start_str), end_str); if (!end) { - log_warn(LD_DIR,"couldn't find end of hashed material \"%s\"",end_str); + log_fn(log_severity,LD_DIR, + "couldn't find end of hashed material \"%s\"",end_str); return -1; } end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str)); if (!end) { - log_warn(LD_DIR,"couldn't find EOL"); + log_fn(log_severity,LD_DIR, + "couldn't find EOL"); return -1; } ++end; @@ -4517,7 +4562,7 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, digest_algorithm_t alg) { const char *start=NULL, *end=NULL; - if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN, &start,&end)<0) return -1; @@ -4554,7 +4599,7 @@ router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests, const char *end_str, char end_c) { const char *start=NULL, *end=NULL; - if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN, &start,&end)<0) return -1; diff --git a/src/or/routerparse.h b/src/or/routerparse.h index e8d71b6dc9..088f773c5e 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -16,6 +16,11 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests); +int router_get_networkstatus_v3_signed_boundaries(const char *s, + const char **start_out, + const char **end_out); +int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, + const char *s); int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest); #define DIROBJ_MAX_SIG_LEN 256 char *router_get_dirobj_signature(const char *digest, |