diff options
Diffstat (limited to 'src/or/directory.c')
| -rw-r--r-- | src/or/directory.c | 4177 |
1 files changed, 2915 insertions, 1262 deletions
diff --git a/src/or/directory.c b/src/or/directory.c index f285e4c6ed..ddb86cd9a4 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1,21 +1,32 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#define DIRECTORY_PRIVATE + #include "or.h" #include "backtrace.h" +#include "bridges.h" #include "buffers.h" #include "circuitbuild.h" #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "conscache.h" +#include "consdiff.h" +#include "consdiffmgr.h" #include "control.h" +#include "compat.h" #include "directory.h" #include "dirserv.h" #include "dirvote.h" #include "entrynodes.h" #include "geoip.h" +#include "hs_cache.h" +#include "hs_common.h" +#include "hs_control.h" +#include "hs_client.h" #include "main.h" #include "microdesc.h" #include "networkstatus.h" @@ -33,16 +44,45 @@ #include "shared_random.h" #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) -#ifndef OPENBSD +#if !defined(OpenBSD) #include <malloc.h> #endif #endif /** * \file directory.c - * \brief Code to send and fetch directories and router - * descriptors via HTTP. Directories use dirserv.c to generate the - * results; clients use routers.c to parse them. + * \brief Code to send and fetch information from directory authorities and + * caches via HTTP. + * + * Directory caches and authorities use dirserv.c to generate the results of a + * query and stream them to the connection; clients use routerparse.c to parse + * them. + * + * Every directory request has a dir_connection_t on the client side and on + * the server side. In most cases, the dir_connection_t object is a linked + * connection, tunneled through an edge_connection_t so that it can be a + * stream on the Tor network. The only non-tunneled connections are those + * that are used to upload material (descriptors and votes) to authorities. + * Among tunneled connections, some use one-hop circuits, and others use + * multi-hop circuits for anonymity. + * + * Directory requests are launched by calling + * directory_initiate_request(). This + * launch the connection, will construct an HTTP request with + * directory_send_command(), send the and wait for a response. The client + * later handles the response with connection_dir_client_reached_eof(), + * which passes the information received to another part of Tor. + * + * On the server side, requests are read in directory_handle_command(), + * which dispatches first on the request type (GET or POST), and then on + * the URL requested. GET requests are processed with a table-based + * dispatcher in url_table[]. The process of handling larger GET requests + * is complicated because we need to avoid allocating a copy of all the + * data to be sent to the client in one huge buffer. Instead, we spool the + * data into the buffer using logic in connection_dirserv_flushed_some() in + * dirserv.c. (TODO: If we extended buf.c to have a zero-copy + * reference-based buffer type, we could remove most of that code, at the + * cost of a bit more reference counting.) **/ /* In-points to directory.c: @@ -62,12 +102,9 @@ * connection_finished_connecting() in connection.c */ static void directory_send_command(dir_connection_t *conn, - int purpose, int direct, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); -static int directory_handle_command(dir_connection_t *conn); + int direct, + const directory_request_t *request); static int body_is_plausible(const char *body, size_t body_len, int purpose); -static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); @@ -80,21 +117,10 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int was_extrainfo, int was_descriptor_digests); static void dir_microdesc_download_failed(smartlist_t *failed, - int status_code); -static int client_likes_consensus(networkstatus_t *v, const char *want_url); - -static void directory_initiate_command_rend( - const tor_addr_port_t *or_addr_port, - const tor_addr_port_t *dir_addr_port, - const char *digest, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query); + int status_code, + const char *dir_id); +static int client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url); static void connection_dir_close_consensus_fetches( dir_connection_t *except_this_one, const char *resource); @@ -106,6 +132,7 @@ static void connection_dir_close_consensus_fetches( #define ALLOW_DIRECTORY_TIME_SKEW (30*60) #define X_ADDRESS_HEADER "X-Your-Address-Is: " +#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: " /** HTTP cache control: how long do we tell proxies they can cache each * kind of document we serve? */ @@ -120,29 +147,58 @@ static void connection_dir_close_consensus_fetches( /********* END VARIABLES ************/ -/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's - * fetching descriptors, it's fetching them for <b>router_purpose</b>) - * must use an anonymous connection to a directory. */ +/** Return false if the directory purpose <b>dir_purpose</b> + * does not require an anonymous (three-hop) connection. + * + * Return true 1) by default, 2) if all directory actions have + * specifically been configured to be over an anonymous connection, + * or 3) if the router is a bridge */ int -purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) +purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource) { if (get_options()->AllDirActionsPrivate) return 1; - if (router_purpose == ROUTER_PURPOSE_BRIDGE) + + if (router_purpose == ROUTER_PURPOSE_BRIDGE) { + if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC + && resource && !strcmp(resource, "authority.z")) { + /* We are asking a bridge for its own descriptor. That doesn't need + anonymity. */ + return 0; + } + /* Assume all other bridge stuff needs anonymity. */ return 1; /* if no circuits yet, this might break bootstrapping, but it's * needed to be safe. */ - if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR || - dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || - dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES || - dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE || - dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES || - dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS || - dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE || - dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC || - dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO || - dir_purpose == DIR_PURPOSE_FETCH_MICRODESC) - return 0; - return 1; + } + + switch (dir_purpose) + { + case DIR_PURPOSE_UPLOAD_DIR: + case DIR_PURPOSE_UPLOAD_VOTE: + case DIR_PURPOSE_UPLOAD_SIGNATURES: + case DIR_PURPOSE_FETCH_STATUS_VOTE: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_CONSENSUS: + case DIR_PURPOSE_FETCH_CERTIFICATE: + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + case DIR_PURPOSE_FETCH_MICRODESC: + return 0; + case DIR_PURPOSE_HAS_FETCHED_HSDESC: + case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2: + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + case DIR_PURPOSE_FETCH_RENDDESC_V2: + case DIR_PURPOSE_FETCH_HSDESC: + case DIR_PURPOSE_UPLOAD_HSDESC: + return 1; + case DIR_PURPOSE_SERVER: + default: + log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d", + dir_purpose, router_purpose); + tor_assert_nonfatal_unreached(); + return 1; /* Assume it needs anonymity; better safe than sorry. */ + } } /** Return a newly allocated string describing <b>auth</b>. Only describes @@ -193,6 +249,10 @@ dir_conn_purpose_to_string(int purpose) return "hidden-service v2 descriptor fetch"; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: return "hidden-service v2 descriptor upload"; + case DIR_PURPOSE_FETCH_HSDESC: + return "hidden-service descriptor fetch"; + case DIR_PURPOSE_UPLOAD_HSDESC: + return "hidden-service descriptor upload"; case DIR_PURPOSE_FETCH_MICRODESC: return "microdescriptor fetch"; } @@ -347,7 +407,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, log_info(LD_DIR, "Uploading an extrainfo too (length %d)", (int) extrainfo_len); } - if (purpose_needs_anonymity(dir_purpose, router_purpose)) { + if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) { indirection = DIRIND_ANONYMOUS; } else if (!fascist_firewall_allows_dir_server(ds, FIREWALL_DIR_CONNECTION, @@ -359,10 +419,14 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, } else { indirection = DIRIND_DIRECT_CONN; } - directory_initiate_command_routerstatus(rs, dir_purpose, - router_purpose, - indirection, - NULL, payload, upload_len, 0); + + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_payload(req, payload, upload_len); + directory_initiate_request(req); + directory_request_free(req); } SMARTLIST_FOREACH_END(ds); if (!found) { char *s = authdir_type_to_string(type); @@ -380,10 +444,9 @@ should_use_directory_guards(const or_options_t *options) /* Public (non-bridge) servers never use directory guards. */ if (public_server_mode(options)) return 0; - /* If guards are disabled, or directory guards are disabled, we can't - * use directory guards. + /* If guards are disabled, we can't use directory guards. */ - if (!options->UseEntryGuards || !options->UseEntryGuardsAsDirGuards) + if (!options->UseEntryGuards) return 0; /* If we're configured to fetch directory info aggressively or of a * nonstandard type, don't use directory guards. */ @@ -398,7 +461,8 @@ should_use_directory_guards(const or_options_t *options) * information of type <b>type</b>, and return its routerstatus. */ static const routerstatus_t * directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, - uint8_t dir_purpose) + uint8_t dir_purpose, + circuit_guard_state_t **guard_state_out) { const routerstatus_t *rs = NULL; const or_options_t *options = get_options(); @@ -407,7 +471,7 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, log_warn(LD_BUG, "Called when we have UseBridges set."); if (should_use_directory_guards(options)) { - const node_t *node = choose_random_dirguard(type); + const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out); if (node) rs = node->rs; } else { @@ -423,13 +487,94 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, return rs; } +/** + * Set the extra fields in <b>req</b> that are used when requesting a + * consensus of type <b>resource</b>. + * + * Right now, these fields are if-modified-since and x-or-diff-from-consensus. + */ +static void +dir_consensus_request_set_additional_headers(directory_request_t *req, + const char *resource) +{ + time_t if_modified_since = 0; + uint8_t or_diff_from[DIGEST256_LEN]; + int or_diff_from_is_set = 0; + + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ + const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180; + const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72; + const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0; + const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192; + const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] = + "try-diff-for-consensus-newer-than"; + + int flav = FLAV_NS; + if (resource) + flav = networkstatus_parse_flavor_name(resource); + + int32_t max_age_for_diff = 3600 * + networkstatus_get_param(NULL, + TRY_DIFF_FOR_CONSENSUS_NEWER_NAME, + DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER, + MIN_TRY_DIFF_FOR_CONSENSUS_NEWER, + MAX_TRY_DIFF_FOR_CONSENSUS_NEWER); + + if (flav != -1) { + /* IF we have a parsed consensus of this type, we can do an + * if-modified-time based on it. */ + networkstatus_t *v; + v = networkstatus_get_latest_consensus_by_flavor(flav); + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + if (v->valid_after >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } + } else { + /* Otherwise it might be a consensus we don't parse, but which we + * do cache. Look at the cached copy, perhaps. */ + cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ + if (cd) { + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (cd->published >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } + } + + if (if_modified_since > 0) + directory_request_set_if_modified_since(req, if_modified_since); + if (or_diff_from_is_set) { + char hex[HEX_DIGEST256_LEN + 1]; + base16_encode(hex, sizeof(hex), + (const char*)or_diff_from, sizeof(or_diff_from)); + directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex); + } +} + /** Start a connection to a random running directory server, using * connection purpose <b>dir_purpose</b>, intending to fetch descriptors * of purpose <b>router_purpose</b>, and requesting <b>resource</b>. * Use <b>pds_flags</b> as arguments to router_pick_directory_server() * or router_pick_trusteddirserver(). */ -MOCK_IMPL(void, directory_get_from_dirserver, ( +MOCK_IMPL(void, +directory_get_from_dirserver,( uint8_t dir_purpose, uint8_t router_purpose, const char *resource, @@ -441,52 +586,17 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( int prefer_authority = (directory_fetches_from_authorities(options) || want_authority == DL_WANT_AUTHORITY); int require_authority = 0; - int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose); + int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose, + resource); dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource); - time_t if_modified_since = 0; if (type == NO_DIRINFO) return; - if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - int flav = FLAV_NS; - networkstatus_t *v; - if (resource) - flav = networkstatus_parse_flavor_name(resource); - - /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus - * period of 1 hour. - */ -#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) - if (flav != -1) { - /* IF we have a parsed consensus of this type, we can do an - * if-modified-time based on it. */ - v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) { - /* In networks with particularly short V3AuthVotingIntervals, - * ask for the consensus if it's been modified since half the - * V3AuthVotingInterval of the most recent consensus. */ - time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; - if (v->fresh_until > v->valid_after - && ims_delay > (v->fresh_until - v->valid_after)/2) { - ims_delay = (v->fresh_until - v->valid_after)/2; - } - if_modified_since = v->valid_after + ims_delay; - } - } else { - /* Otherwise it might be a consensus we don't parse, but which we - * do cache. Look at the cached copy, perhaps. */ - cached_dir_t *cd = dirserv_get_consensus(resource); - /* We have no method of determining the voting interval from an - * unparsed consensus, so we use the default. */ - if (cd) - if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; - } - } - if (!options->FetchServerDescriptors) return; + circuit_guard_state_t *guard_state = NULL; if (!get_via_tor) { if (options->UseBridges && !(type & BRIDGE_DIRINFO)) { /* We want to ask a running bridge for which we have a descriptor. @@ -495,25 +605,34 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( * sort of dir fetch we'll be doing, so it won't return a bridge * that can't answer our question. */ - const node_t *node = choose_random_dirguard(type); + const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state); if (node && node->ri) { /* every bridge has a routerinfo. */ routerinfo_t *ri = node->ri; /* clients always make OR connections to bridges */ tor_addr_port_t or_ap; + directory_request_t *req = directory_request_new(dir_purpose); /* we are willing to use a non-preferred address if we need to */ fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &or_ap); - directory_initiate_command(&or_ap.addr, or_ap.port, - NULL, 0, /*no dirport*/ - ri->cache_info.identity_digest, - dir_purpose, - router_purpose, - DIRIND_ONEHOP, - resource, NULL, 0, if_modified_since); - } else + directory_request_set_or_addr_port(req, &or_ap); + directory_request_set_directory_id_digest(req, + ri->cache_info.identity_digest); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); + } else { + if (guard_state) { + entry_guard_cancel(&guard_state); + } log_notice(LD_DIR, "Ignoring directory request, since no bridge " "nodes are available yet."); + } + return; } else { if (prefer_authority || (type & BRIDGE_DIRINFO)) { @@ -544,9 +663,9 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( } } if (!rs && !(type & BRIDGE_DIRINFO)) { - /* */ rs = directory_pick_generic_dirserver(type, pds_flags, - dir_purpose); + dir_purpose, + &guard_state); if (!rs) get_via_tor = 1; /* last resort: try routing it via Tor */ } @@ -565,17 +684,23 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( if (rs) { const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; - directory_initiate_command_routerstatus(rs, dir_purpose, - router_purpose, - indirection, - resource, NULL, 0, - if_modified_since); + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + if (guard_state) + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); } else { log_notice(LD_DIR, "While fetching directory info, " "no running dirservers known. Will try again later. " "(purpose %d)", dir_purpose); - if (!purpose_needs_anonymity(dir_purpose, router_purpose)) { + if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) { /* remember we tried them all and failed. */ directory_all_unreachable(time(NULL)); } @@ -595,15 +720,17 @@ directory_get_from_all_authorities(uint8_t dir_purpose, SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(), dir_server_t *, ds) { - routerstatus_t *rs; if (router_digest_is_me(ds->digest)) continue; if (!(ds->type & V3_DIRINFO)) continue; - rs = &ds->fake_status; - directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose, - DIRIND_ONEHOP, resource, NULL, - 0, 0); + const routerstatus_t *rs = &ds->fake_status; + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); } SMARTLIST_FOREACH_END(ds); } @@ -703,106 +830,6 @@ directory_choose_address_routerstatus(const routerstatus_t *status, return 0; } -/** Same as directory_initiate_command_routerstatus(), but accepts - * rendezvous data to fetch a hidden service descriptor. */ -void -directory_initiate_command_routerstatus_rend(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query) -{ - const or_options_t *options = get_options(); - const node_t *node; - tor_addr_port_t use_or_ap, use_dir_ap; - const int anonymized_connection = dirind_is_anon(indirection); - - tor_assert(status != NULL); - - node = node_get_by_id(status->identity_digest); - - /* XXX The below check is wrong: !node means it's not in the consensus, - * but we haven't checked if we have a descriptor for it -- and also, - * we only care about the descriptor if it's a begindir-style anonymized - * connection. */ - if (!node && anonymized_connection) { - log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " - "don't have its router descriptor.", - routerstatus_describe(status)); - return; - } - - if (options->ExcludeNodes && options->StrictNodes && - routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { - log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but " - "it's in our ExcludedNodes list and StrictNodes is set. " - "Skipping. This choice might make your Tor not work.", - routerstatus_describe(status), - dir_conn_purpose_to_string(dir_purpose)); - return; - } - - /* At this point, if we are a client making a direct connection to a - * directory server, we have selected a server that has at least one address - * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This - * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if - * possible. (If UseBridges is set, clients always use IPv6, and prefer it - * by default.) - * - * Now choose an address that we can use to connect to the directory server. - */ - if (directory_choose_address_routerstatus(status, indirection, &use_or_ap, - &use_dir_ap) < 0) { - return; - } - - /* We don't retry the alternate OR/Dir address for the same directory if - * the address we choose fails (#6772). - * Instead, we'll retry another directory on failure. */ - - directory_initiate_command_rend(&use_or_ap, &use_dir_ap, - status->identity_digest, - dir_purpose, router_purpose, - indirection, resource, - payload, payload_len, if_modified_since, - rend_query); -} - -/** Launch a new connection to the directory server <b>status</b> to - * upload or download a server or rendezvous - * descriptor. <b>dir_purpose</b> determines what - * kind of directory connection we're launching, and must be one of - * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b> - * specifies the descriptor purposes we have in mind (currently only - * used for FETCH_DIR). - * - * When uploading, <b>payload</b> and <b>payload_len</b> determine the content - * of the HTTP post. Otherwise, <b>payload</b> should be NULL. - * - * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we - * want to fetch. - */ -MOCK_IMPL(void, directory_initiate_command_routerstatus, - (const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since)) -{ - directory_initiate_command_routerstatus_rend(status, dir_purpose, - router_purpose, - indirection, resource, - payload, payload_len, - if_modified_since, NULL); -} - /** Return true iff <b>conn</b> is the client side of a directory connection * we launched to ourself in order to determine the reachability of our * dir_port. */ @@ -828,6 +855,11 @@ directory_conn_is_self_reachability_test(dir_connection_t *conn) static void connection_dir_request_failed(dir_connection_t *conn) { + if (conn->guard_state) { + /* We haven't seen a success on this guard state, so consider it to have + * failed. */ + entry_guard_failed(&conn->guard_state); + } if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } @@ -983,6 +1015,7 @@ directory_must_use_begindir(const or_options_t *options) /** Evaluate the situation and decide if we should use an encrypted * "begindir-style" connection for this directory request. + * 0) If there is no DirPort, yes. * 1) If or_port is 0, or it's a direct conn and or_port is firewalled * or we're a dir mirror, no. * 2) If we prefer to avoid begindir conns, and we're not fetching or @@ -993,15 +1026,24 @@ directory_must_use_begindir(const or_options_t *options) */ static int directory_command_should_use_begindir(const or_options_t *options, - const tor_addr_t *addr, - int or_port, uint8_t router_purpose, - dir_indirection_t indirection, + const directory_request_t *req, const char **reason) { - (void) router_purpose; + const tor_addr_t *or_addr = &req->or_addr_port.addr; + //const tor_addr_t *dir_addr = &req->dir_addr_port.addr; + const int or_port = req->or_addr_port.port; + const int dir_port = req->dir_addr_port.port; + + const dir_indirection_t indirection = req->indirection; + tor_assert(reason); *reason = NULL; + /* Reasons why we must use begindir */ + if (!dir_port) { + *reason = "(using begindir - directory with no DirPort)"; + return 1; /* We don't know a DirPort -- must begindir. */ + } /* Reasons why we can't possibly use begindir */ if (!or_port) { *reason = "directory with unknown ORPort"; @@ -1014,7 +1056,7 @@ directory_command_should_use_begindir(const or_options_t *options, } if (indirection == DIRIND_ONEHOP) { /* We're firewalled and want a direct OR connection */ - if (!fascist_firewall_allows_address_addr(addr, or_port, + if (!fascist_firewall_allows_address_addr(or_addr, or_port, FIREWALL_OR_CONNECTION, 0, 0)) { *reason = "ORPort not reachable"; return 0; @@ -1033,79 +1075,320 @@ directory_command_should_use_begindir(const or_options_t *options, return 1; } -/** Helper for directory_initiate_command_rend: send the - * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>, - * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose - * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and - * <b>router_purpose</b>, making an (in)direct connection as specified in - * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of - * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>. +/** + * Create and return a new directory_request_t with purpose + * <b>dir_purpose</b>. + */ +directory_request_t * +directory_request_new(uint8_t dir_purpose) +{ + tor_assert(dir_purpose >= DIR_PURPOSE_MIN_); + tor_assert(dir_purpose <= DIR_PURPOSE_MAX_); + tor_assert(dir_purpose != DIR_PURPOSE_SERVER); + tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2); + tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_HSDESC); + + directory_request_t *result = tor_malloc_zero(sizeof(*result)); + tor_addr_make_null(&result->or_addr_port.addr, AF_INET); + result->or_addr_port.port = 0; + tor_addr_make_null(&result->dir_addr_port.addr, AF_INET); + result->dir_addr_port.port = 0; + result->dir_purpose = dir_purpose; + result->router_purpose = ROUTER_PURPOSE_GENERAL; + result->indirection = DIRIND_ONEHOP; + return result; +} +/** + * Release all resources held by <b>req</b>. */ void -directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, - const tor_addr_t *dir_addr, uint16_t dir_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since) +directory_request_free_(directory_request_t *req) { - tor_addr_port_t or_ap, dir_ap; + if (req == NULL) + return; + config_free_lines(req->additional_headers); + tor_free(req); +} +/** + * Set the address and OR port to use for this directory request. If there is + * no OR port, we'll have to connect over the dirport. (If there are both, + * the indirection setting determines which to use.) + */ +void +directory_request_set_or_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->or_addr_port, p, sizeof(*p)); +} +/** + * Set the address and dirport to use for this directory request. If there + * is no dirport, we'll have to connect over the OR port. (If there are both, + * the indirection setting determines which to use.) + */ +void +directory_request_set_dir_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->dir_addr_port, p, sizeof(*p)); +} +/** + * Set the RSA identity digest of the directory to use for this directory + * request. + */ +void +directory_request_set_directory_id_digest(directory_request_t *req, + const char *digest) +{ + memcpy(req->digest, digest, DIGEST_LEN); +} +/** + * Set the router purpose associated with uploaded and downloaded router + * descriptors and extrainfo documents in this directory request. The purpose + * must be one of ROUTER_PURPOSE_GENERAL (the default) or + * ROUTER_PURPOSE_BRIDGE. + */ +void +directory_request_set_router_purpose(directory_request_t *req, + uint8_t router_purpose) +{ + tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL || + router_purpose == ROUTER_PURPOSE_BRIDGE); + // assert that it actually makes sense to set this purpose, given + // the dir_purpose. + req->router_purpose = router_purpose; +} +/** + * Set the indirection to be used for the directory request. The indirection + * parameter configures whether to connect to a DirPort or ORPort, and whether + * to anonymize the connection. DIRIND_ONEHOP (use ORPort, don't anonymize) + * is the default. See dir_indirection_t for more information. + */ +void +directory_request_set_indirection(directory_request_t *req, + dir_indirection_t indirection) +{ + req->indirection = indirection; +} - /* Use the null tor_addr and 0 port if the address or port isn't valid. */ - if (tor_addr_port_is_valid(or_addr, or_port, 0)) { - tor_addr_copy(&or_ap.addr, or_addr); - or_ap.port = or_port; - } else { - /* the family doesn't matter here, so make it IPv4 */ - tor_addr_make_null(&or_ap.addr, AF_INET); - or_ap.port = or_port = 0; - } +/** + * Set a pointer to the resource to request from a directory. Different + * request types use resources to indicate different components of their URL. + * Note that only an alias to <b>resource</b> is stored, so the + * <b>resource</b> must outlive the request. + */ +void +directory_request_set_resource(directory_request_t *req, + const char *resource) +{ + req->resource = resource; +} +/** + * Set a pointer to the payload to include with this directory request, along + * with its length. Note that only an alias to <b>payload</b> is stored, so + * the <b>payload</b> must outlive the request. + */ +void +directory_request_set_payload(directory_request_t *req, + const char *payload, + size_t payload_len) +{ + tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose)); - if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) { - tor_addr_copy(&dir_ap.addr, dir_addr); - dir_ap.port = dir_port; - } else { - /* the family doesn't matter here, so make it IPv4 */ - tor_addr_make_null(&dir_ap.addr, AF_INET); - dir_ap.port = dir_port = 0; + req->payload = payload; + req->payload_len = payload_len; +} +/** + * Set an if-modified-since date to send along with the request. The + * default is 0 (meaning, send no if-modified-since header). + */ +void +directory_request_set_if_modified_since(directory_request_t *req, + time_t if_modified_since) +{ + req->if_modified_since = if_modified_since; +} + +/** Include a header of name <b>key</b> with content <b>val</b> in the + * request. Neither may include newlines or other odd characters. Their + * ordering is not currently guaranteed. + * + * Note that, as elsewhere in this module, header keys include a trailing + * colon and space. + */ +void +directory_request_add_header(directory_request_t *req, + const char *key, + const char *val) +{ + config_line_prepend(&req->additional_headers, key, val); +} +/** + * Set an object containing HS data to be associated with this request. Note + * that only an alias to <b>query</b> is stored, so the <b>query</b> object + * must outlive the request. + */ +void +directory_request_set_rend_query(directory_request_t *req, + const rend_data_t *query) +{ + if (query) { + tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 || + req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); + } + req->rend_query = query; +} +/** + * Set an object containing HS connection identifier to be associated with + * this request. Note that only an alias to <b>ident</b> is stored, so the + * <b>ident</b> object must outlive the request. + */ +void +directory_request_upload_set_hs_ident(directory_request_t *req, + const hs_ident_dir_conn_t *ident) +{ + if (ident) { + tor_assert(req->dir_purpose == DIR_PURPOSE_UPLOAD_HSDESC); + } + req->hs_ident = ident; +} +/** + * Set an object containing HS connection identifier to be associated with + * this fetch request. Note that only an alias to <b>ident</b> is stored, so + * the <b>ident</b> object must outlive the request. + */ +void +directory_request_fetch_set_hs_ident(directory_request_t *req, + const hs_ident_dir_conn_t *ident) +{ + if (ident) { + tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_HSDESC); } + req->hs_ident = ident; +} +/** Set a static circuit_guard_state_t object to affliate with the request in + * <b>req</b>. This object will receive notification when the attempt to + * connect to the guard either succeeds or fails. */ +void +directory_request_set_guard_state(directory_request_t *req, + circuit_guard_state_t *state) +{ + req->guard_state = state; +} - directory_initiate_command_rend(&or_ap, &dir_ap, - digest, dir_purpose, - router_purpose, indirection, - resource, payload, payload_len, - if_modified_since, NULL); +/** + * Internal: Return true if any information for contacting the directory in + * <b>req</b> has been set, other than by the routerstatus. */ +static int +directory_request_dir_contact_info_specified(const directory_request_t *req) +{ + /* We only check for ports here, since we don't use an addr unless the port + * is set */ + return (req->or_addr_port.port || + req->dir_addr_port.port || + ! tor_digest_is_zero(req->digest)); } -/** Return non-zero iff a directory connection with purpose - * <b>dir_purpose</b> reveals sensitive information about a Tor - * instance's client activities. (Such connections must be performed - * through normal three-hop Tor circuits.) */ -int -is_sensitive_dir_purpose(uint8_t dir_purpose) +/** + * Set the routerstatus to use for the directory associated with this + * request. If this option is set, then no other function to set the + * directory's address or identity should be called. + */ +void +directory_request_set_routerstatus(directory_request_t *req, + const routerstatus_t *status) { - return ((dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2) || - (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) || - (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)); + req->routerstatus = status; } +/** + * Helper: update the addresses, ports, and identities in <b>req</b> + * from the routerstatus object in <b>req</b>. Return 0 on success. + * On failure, warn and return -1. + */ +static int +directory_request_set_dir_from_routerstatus(directory_request_t *req) -/** Same as directory_initiate_command(), but accepts rendezvous data to - * fetch a hidden service descriptor, and takes its address & port arguments - * as tor_addr_port_t. */ -static void -directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, - const tor_addr_port_t *dir_addr_port, - const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query) { - tor_assert(or_addr_port); - tor_assert(dir_addr_port); + const routerstatus_t *status = req->routerstatus; + if (BUG(status == NULL)) + return -1; + const or_options_t *options = get_options(); + const node_t *node; + tor_addr_port_t use_or_ap, use_dir_ap; + const int anonymized_connection = dirind_is_anon(req->indirection); + + tor_assert(status != NULL); + + node = node_get_by_id(status->identity_digest); + + /* XXX The below check is wrong: !node means it's not in the consensus, + * but we haven't checked if we have a descriptor for it -- and also, + * we only care about the descriptor if it's a begindir-style anonymized + * connection. */ + if (!node && anonymized_connection) { + log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " + "don't have its router descriptor.", + routerstatus_describe(status)); + return -1; + } + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { + log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping. This choice might make your Tor not work.", + routerstatus_describe(status), + dir_conn_purpose_to_string(req->dir_purpose)); + return -1; + } + + /* At this point, if we are a client making a direct connection to a + * directory server, we have selected a server that has at least one address + * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This + * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if + * possible. (If UseBridges is set, clients always use IPv6, and prefer it + * by default.) + * + * Now choose an address that we can use to connect to the directory server. + */ + if (directory_choose_address_routerstatus(status, + req->indirection, &use_or_ap, + &use_dir_ap) < 0) { + return -1; + } + + directory_request_set_or_addr_port(req, &use_or_ap); + directory_request_set_dir_addr_port(req, &use_dir_ap); + directory_request_set_directory_id_digest(req, status->identity_digest); + return 0; +} + +/** + * Launch the provided directory request, configured in <b>request</b>. + * After this function is called, you can free <b>request</b>. + */ +MOCK_IMPL(void, +directory_initiate_request,(directory_request_t *request)) +{ + tor_assert(request); + if (request->routerstatus) { + tor_assert_nonfatal( + ! directory_request_dir_contact_info_specified(request)); + if (directory_request_set_dir_from_routerstatus(request) < 0) { + return; + } + } + + const tor_addr_port_t *or_addr_port = &request->or_addr_port; + const tor_addr_port_t *dir_addr_port = &request->dir_addr_port; + const char *digest = request->digest; + const uint8_t dir_purpose = request->dir_purpose; + const uint8_t router_purpose = request->router_purpose; + const dir_indirection_t indirection = request->indirection; + const char *resource = request->resource; + const rend_data_t *rend_query = request->rend_query; + const hs_ident_dir_conn_t *hs_ident = request->hs_ident; + circuit_guard_state_t *guard_state = request->guard_state; + tor_assert(or_addr_port->port || dir_addr_port->port); tor_assert(digest); @@ -1115,10 +1398,9 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, const char *begindir_reason = NULL; /* Should the connection be to a relay's OR port (and inside that we will * send our directory request)? */ - const int use_begindir = directory_command_should_use_begindir(options, - &or_addr_port->addr, or_addr_port->port, - router_purpose, indirection, - &begindir_reason); + const int use_begindir = + directory_command_should_use_begindir(options, request, &begindir_reason); + /* Will the connection go via a three-hop Tor circuit? Note that this * is separate from whether it will use_begindir. */ const int anonymized_connection = dirind_is_anon(indirection); @@ -1137,7 +1419,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); - if (is_sensitive_dir_purpose(dir_purpose)) { + if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) { tor_assert(anonymized_connection || rend_non_anonymous_mode_enabled(options)); } @@ -1163,9 +1445,9 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, if (!port || tor_addr_is_null(&addr)) { static int logged_backtrace = 0; log_warn(LD_DIR, - "Cannot make an outgoing %sconnection without %sPort.", + "Cannot make an outgoing %sconnection without a remote %sPort.", use_begindir ? "begindir " : "", - use_begindir ? "an OR" : "a Dir"); + use_begindir ? "OR" : "Dir"); if (!logged_backtrace) { log_backtrace(LOG_INFO, LD_BUG, "Address came from"); logged_backtrace = 1; @@ -1192,8 +1474,16 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, conn->dirconn_direct = !anonymized_connection; /* copy rendezvous data, if any */ - if (rend_query) + if (rend_query) { + /* We can't have both v2 and v3+ identifier. */ + tor_assert_nonfatal(!hs_ident); conn->rend_data = rend_data_dup(rend_query); + } + if (hs_ident) { + /* We can't have both v2 and v3+ identifier. */ + tor_assert_nonfatal(!rend_query); + conn->hs_ident = hs_ident_dir_conn_dup(hs_ident); + } if (!anonymized_connection && !use_begindir) { /* then we want to connect to dirport directly */ @@ -1203,6 +1493,11 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, port = options->HTTPProxyPort; } + // In this case we should not have picked a directory guard. + if (BUG(guard_state)) { + entry_guard_cancel(&guard_state); + } + switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, port, &socket_error)) { case -1: @@ -1214,9 +1509,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, /* fall through */ case 0: /* queue the command on the outbuf */ - directory_send_command(conn, dir_purpose, 1, resource, - payload, payload_len, - if_modified_since); + directory_send_command(conn, 1, request); connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ @@ -1239,6 +1532,14 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, else if (anonymized_connection && !use_begindir) rep_hist_note_used_port(time(NULL), conn->base_.port); + // In this case we should not have a directory guard; we'll + // get a regular guard later when we build the circuit. + if (BUG(anonymized_connection && guard_state)) { + entry_guard_cancel(&guard_state); + } + + conn->guard_state = guard_state; + /* make an AP connection * populate it and add it at the right state * hook up both sides @@ -1262,9 +1563,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, } conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; /* queue the command on the outbuf */ - directory_send_command(conn, dir_purpose, 0, resource, - payload, payload_len, - if_modified_since); + directory_send_command(conn, 0, request); connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(ENTRY_TO_CONN(linked_conn)); @@ -1274,7 +1573,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, /** Return true iff anything we say on <b>conn</b> is being encrypted before * we send it to the client/server. */ int -connection_dir_is_encrypted(dir_connection_t *conn) +connection_dir_is_encrypted(const dir_connection_t *conn) { /* Right now it's sufficient to see if conn is or has been linked, since * the only thing it could be linked to is an edge connection on a @@ -1367,15 +1666,23 @@ copy_ipv6_address(char* destination, const char* source, size_t len, } } -/** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command(). +/** Queue an appropriate HTTP command for <b>request</b> on + * <b>conn</b>-\>outbuf. If <b>direct</b> is true, we're making a + * non-anonymized connection to the dirport. */ static void directory_send_command(dir_connection_t *conn, - int purpose, int direct, const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since) + const int direct, + const directory_request_t *req) { + tor_assert(req); + const int purpose = req->dir_purpose; + const char *resource = req->resource; + const char *payload = req->payload; + const size_t payload_len = req->payload_len; + const time_t if_modified_since = req->if_modified_since; + const int anonymized_connection = dirind_is_anon(req->indirection); + char proxystring[256]; char hoststring[128]; /* NEEDS to be the same size hoststring. @@ -1383,7 +1690,10 @@ directory_send_command(dir_connection_t *conn, char decorated_address[128]; smartlist_t *headers = smartlist_new(); char *url; + char *accept_encoding; + size_t url_len; char request[8192]; + size_t request_len, total_request_len = 0; const char *httpcommand = NULL; tor_assert(conn); @@ -1437,6 +1747,22 @@ directory_send_command(dir_connection_t *conn, proxystring[0] = 0; } + if (! anonymized_connection) { + /* Add Accept-Encoding. */ + accept_encoding = accept_encoding_header(); + smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n", + accept_encoding); + tor_free(accept_encoding); + } + + /* Add additional headers, if any */ + { + config_line_t *h; + for (h = req->additional_headers; h; h = h->next) { + smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value); + } + } + switch (purpose) { case DIR_PURPOSE_FETCH_CONSENSUS: /* resource is optional. If present, it's a flavor name */ @@ -1485,9 +1811,10 @@ directory_send_command(dir_connection_t *conn, tor_assert(payload); httpcommand = "POST"; url = tor_strdup("/tor/"); - if (why) { - smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why); + if (!why) { + why = "for no reason at all"; } + smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why); break; } case DIR_PURPOSE_UPLOAD_VOTE: @@ -1509,12 +1836,25 @@ directory_send_command(dir_connection_t *conn, httpcommand = "GET"; tor_asprintf(&url, "/tor/rendezvous2/%s", resource); break; + case DIR_PURPOSE_FETCH_HSDESC: + tor_assert(resource); + tor_assert(strlen(resource) <= ED25519_BASE64_LEN); + tor_assert(!payload); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/hs/3/%s", resource); + break; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: tor_assert(!resource); tor_assert(payload); httpcommand = "POST"; url = tor_strdup("/tor/rendezvous2/publish"); break; + case DIR_PURPOSE_UPLOAD_HSDESC: + tor_assert(resource); + tor_assert(payload); + httpcommand = "POST"; + tor_asprintf(&url, "/tor/hs/%s/publish", resource); + break; default: tor_assert(0); return; @@ -1529,8 +1869,14 @@ directory_send_command(dir_connection_t *conn, } tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring); - connection_write_to_buf(request, strlen(request), TO_CONN(conn)); - connection_write_to_buf(url, strlen(url), TO_CONN(conn)); + + request_len = strlen(request); + total_request_len += request_len; + connection_buf_add(request, request_len, TO_CONN(conn)); + + url_len = strlen(url); + total_request_len += url_len; + connection_buf_add(url, url_len, TO_CONN(conn)); tor_free(url); if (!strcmp(httpcommand, "POST") || payload) { @@ -1545,15 +1891,27 @@ directory_send_command(dir_connection_t *conn, tor_free(header); } - connection_write_to_buf(request, strlen(request), TO_CONN(conn)); + request_len = strlen(request); + total_request_len += request_len; + connection_buf_add(request, request_len, TO_CONN(conn)); if (payload) { /* then send the payload afterwards too */ - connection_write_to_buf(payload, payload_len, TO_CONN(conn)); + connection_buf_add(payload, payload_len, TO_CONN(conn)); + total_request_len += payload_len; } SMARTLIST_FOREACH(headers, char *, h, tor_free(h)); smartlist_free(headers); + + log_debug(LD_DIR, + "Sent request to directory server '%s:%d': " + "(purpose: %d, request size: " U64_FORMAT ", " + "payload size: " U64_FORMAT ")", + conn->base_.address, conn->base_.port, + conn->base_.purpose, + U64_PRINTF_ARG(total_request_len), + U64_PRINTF_ARG(payload ? payload_len : 0)); } /** Parse an HTTP request string <b>headers</b> of the form @@ -1568,15 +1926,41 @@ directory_send_command(dir_connection_t *conn, STATIC int parse_http_url(const char *headers, char **url) { + char *command = NULL; + if (parse_http_command(headers, &command, url) < 0) { + return -1; + } + if (strcmpstart(*url, "/tor/")) { + char *new_url = NULL; + tor_asprintf(&new_url, "/tor%s%s", + *url[0] == '/' ? "" : "/", + *url); + tor_free(*url); + *url = new_url; + } + tor_free(command); + return 0; +} + +/** Parse an HTTP request line at the start of a headers string. On failure, + * return -1. On success, set *<b>command_out</b> to a copy of the HTTP + * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and + * return 0. */ +int +parse_http_command(const char *headers, char **command_out, char **url_out) +{ + const char *command, *end_of_command; char *s, *start, *tmp; s = (char *)eat_whitespace_no_nl(headers); if (!*s) return -1; + command = s; s = (char *)find_whitespace(s); /* get past GET/POST */ if (!*s) return -1; + end_of_command = s; s = (char *)eat_whitespace_no_nl(s); if (!*s) return -1; - start = s; /* this is it, assuming it's valid */ + start = s; /* this is the URL, assuming it's valid */ s = (char *)find_whitespace(start); if (!*s) return -1; @@ -1607,13 +1991,8 @@ parse_http_url(const char *headers, char **url) return -1; } - if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */ - *url = tor_malloc(s - start + 5); - strlcpy(*url,"/tor", s-start+5); - strlcat((*url)+4, start, s-start+1); - } else { - *url = tor_strndup(start, s-start); - } + *url_out = tor_memdup_nulterm(start, s-start); + *command_out = tor_memdup_nulterm(command, end_of_command - command); return 0; } @@ -1621,7 +2000,7 @@ parse_http_url(const char *headers, char **url) * <b>which</b>. The key should be given with a terminating colon and space; * this function copies everything after, up to but not including the * following \\r\\n. */ -static char * +char * http_get_header(const char *headers, const char *which) { const char *cp = headers; @@ -1738,16 +2117,15 @@ parse_http_response(const char *headers, int *code, time_t *date, if (!strcmpstart(s, "Content-Encoding: ")) { enc = s+18; break; }); - if (!enc || !strcmp(enc, "identity")) { + + if (enc == NULL) *compression = NO_METHOD; - } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) { - *compression = ZLIB_METHOD; - } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) { - *compression = GZIP_METHOD; - } else { - log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", - escaped(enc)); - *compression = UNKNOWN_METHOD; + else { + *compression = compression_method_get_by_name(enc); + + if (*compression == UNKNOWN_METHOD) + log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", + escaped(enc)); } } SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s)); @@ -1770,15 +2148,15 @@ body_is_plausible(const char *body, size_t len, int purpose) if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { return (!strcmpstart(body,"onion-key")); } - if (1) { - if (!strcmpstart(body,"router") || - !strcmpstart(body,"network-status")) - return 1; - for (i=0;i<32;++i) { - if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i])) - return 0; - } + + if (!strcmpstart(body,"router") || + !strcmpstart(body,"network-status")) + return 1; + for (i=0;i<32;++i) { + if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i])) + return 0; } + return 1; } @@ -1820,6 +2198,144 @@ load_downloaded_routers(const char *body, smartlist_t *which, return added; } +static int handle_response_fetch_certificate(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_status_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_detached_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_desc(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_dir(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_hsdesc(dir_connection_t *, + const response_handler_args_t *); + +static int +dir_client_decompress_response_body(char **bodyp, size_t *bodylenp, + dir_connection_t *conn, + compress_method_t compression, + int anonymized_connection) +{ + int rv = 0; + const char *body = *bodyp; + size_t body_len = *bodylenp; + int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || + conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + + int plausible = body_is_plausible(body, body_len, conn->base_.purpose); + + if (plausible && compression == NO_METHOD) { + return 0; + } + + int severity = LOG_DEBUG; + char *new_body = NULL; + size_t new_len = 0; + const char *description1, *description2; + int want_to_try_both = 0; + int tried_both = 0; + compress_method_t guessed = detect_compression_method(body, body_len); + + description1 = compression_method_get_human_name(compression); + + if (BUG(description1 == NULL)) + description1 = compression_method_get_human_name(UNKNOWN_METHOD); + + if (guessed == UNKNOWN_METHOD && !plausible) + description2 = "confusing binary junk"; + else + description2 = compression_method_get_human_name(guessed); + + /* Tell the user if we don't believe what we're told about compression.*/ + want_to_try_both = (compression == UNKNOWN_METHOD || + guessed != compression); + if (want_to_try_both) { + severity = LOG_PROTOCOL_WARN; + } + + tor_log(severity, LD_HTTP, + "HTTP body from server '%s:%d' was labeled as %s, " + "%s it seems to be %s.%s", + conn->base_.address, conn->base_.port, description1, + guessed != compression?"but":"and", + description2, + (compression>0 && guessed>0 && want_to_try_both)? + " Trying both.":""); + + /* Try declared compression first if we can. + * tor_compress_supports_method() also returns true for NO_METHOD. + * Ensure that the server is not sending us data compressed using a + * compression method that is not allowed for anonymous connections. */ + if (anonymized_connection && + ! allowed_anonymous_connection_compression_method(compression)) { + warn_disallowed_anonymous_compression_method(compression); + rv = -1; + goto done; + } + + if (tor_compress_supports_method(compression)) { + tor_uncompress(&new_body, &new_len, body, body_len, compression, + !allow_partial, LOG_PROTOCOL_WARN); + if (new_body) { + /* We succeeded with the declared compression method. Great! */ + rv = 0; + goto done; + } + } + + /* Okay, if that didn't work, and we think that it was compressed + * differently, try that. */ + if (anonymized_connection && + ! allowed_anonymous_connection_compression_method(guessed)) { + warn_disallowed_anonymous_compression_method(guessed); + rv = -1; + goto done; + } + + if (tor_compress_supports_method(guessed) && + compression != guessed) { + tor_uncompress(&new_body, &new_len, body, body_len, guessed, + !allow_partial, LOG_INFO); + tried_both = 1; + } + /* If we're pretty sure that we have a compressed directory, and + * we didn't manage to uncompress it, then warn and bail. */ + if (!plausible && !new_body) { + log_fn(LOG_PROTOCOL_WARN, LD_HTTP, + "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').", + description1, + tried_both?" and ":"", + tried_both?description2:"", + conn->base_.address, conn->base_.port); + rv = -1; + goto done; + } + + done: + if (new_body) { + if (rv == 0) { + /* success! */ + tor_free(*bodyp); + *bodyp = new_body; + *bodylenp = new_len; + } else { + tor_free(new_body); + } + } + + return rv; +} + /** We are a client, and we've finished reading the server's * response. Parse it and act appropriately. * @@ -1832,21 +2348,26 @@ load_downloaded_routers(const char *body, smartlist_t *which, static int connection_dir_client_reached_eof(dir_connection_t *conn) { - char *body; - char *headers; + char *body = NULL; + char *headers = NULL; char *reason = NULL; size_t body_len = 0; int status_code; time_t date_header = 0; long apparent_skew; compress_method_t compression; - int plausible; int skewed = 0; + int rv; int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - time_t now = time(NULL); - int src_code; + size_t received_bytes; + const int anonymized_connection = + purpose_needs_anonymity(conn->base_.purpose, + conn->router_purpose, + conn->requested_resource); + + received_bytes = connection_get_inbuf_len(TO_CONN(conn)); switch (connection_fetch_from_buf_http(TO_CONN(conn), &headers, MAX_HEADERS_SIZE, @@ -1868,17 +2389,39 @@ connection_dir_client_reached_eof(dir_connection_t *conn) &compression, &reason) < 0) { log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.", conn->base_.address, conn->base_.port); - tor_free(body); tor_free(headers); - return -1; + + rv = -1; + goto done; } if (!reason) reason = tor_strdup("[no reason given]"); - log_debug(LD_DIR, + tor_log(LOG_DEBUG, LD_DIR, "Received response from directory server '%s:%d': %d %s " - "(purpose: %d)", + "(purpose: %d, response size: " U64_FORMAT +#ifdef MEASUREMENTS_21206 + ", data cells received: %d, data cells sent: %d" +#endif + ", compression: %d)", conn->base_.address, conn->base_.port, status_code, - escaped(reason), - conn->base_.purpose); + escaped(reason), conn->base_.purpose, + U64_PRINTF_ARG(received_bytes), +#ifdef MEASUREMENTS_21206 + conn->data_cells_received, conn->data_cells_sent, +#endif + compression); + + if (conn->guard_state) { + /* we count the connection as successful once we can read from it. We do + * not, however, delay use of the circuit here, since it's just for a + * one-hop directory request. */ + /* XXXXprop271 note that this will not do the right thing for other + * waiting circuits that would be triggered by this circuit becoming + * complete/usable. But that's ok, I think. + */ + entry_guard_succeeded(&conn->guard_state); + circuit_guard_state_free(conn->guard_state); + conn->guard_state = NULL; + } /* now check if it's got any hints for us about our IP address. */ if (conn->dirconn_direct) { @@ -1916,540 +2459,886 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. I'll try again soon.", status_code, escaped(reason), conn->base_.address, conn->base_.port); + time_t now = approx_time(); if ((rs = router_get_mutable_consensus_status_by_id(id_digest))) rs->last_dir_503_at = now; if ((ds = router_get_fallback_dirserver_by_digest(id_digest))) ds->fake_status.last_dir_503_at = now; - tor_free(body); tor_free(headers); tor_free(reason); - return -1; + rv = -1; + goto done; } - plausible = body_is_plausible(body, body_len, conn->base_.purpose); - if (compression != NO_METHOD || !plausible) { - char *new_body = NULL; - size_t new_len = 0; - compress_method_t guessed = detect_compression_method(body, body_len); - if (compression == UNKNOWN_METHOD || guessed != compression) { - /* Tell the user if we don't believe what we're told about compression.*/ - const char *description1, *description2; - if (compression == ZLIB_METHOD) - description1 = "as deflated"; - else if (compression == GZIP_METHOD) - description1 = "as gzipped"; - else if (compression == NO_METHOD) - description1 = "as uncompressed"; - else - description1 = "with an unknown Content-Encoding"; - if (guessed == ZLIB_METHOD) - description2 = "deflated"; - else if (guessed == GZIP_METHOD) - description2 = "gzipped"; - else if (!plausible) - description2 = "confusing binary junk"; - else - description2 = "uncompressed"; + if (dir_client_decompress_response_body(&body, &body_len, + conn, compression, anonymized_connection) < 0) { + rv = -1; + goto done; + } - log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, " - "but it seems to be %s.%s", - conn->base_.address, conn->base_.port, description1, - description2, - (compression>0 && guessed>0)?" Trying both.":""); - } - /* Try declared compression first if we can. */ - if (compression == GZIP_METHOD || compression == ZLIB_METHOD) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression, - !allow_partial, LOG_PROTOCOL_WARN); - /* Okay, if that didn't work, and we think that it was compressed - * differently, try that. */ - if (!new_body && - (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && - compression != guessed) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed, - !allow_partial, LOG_PROTOCOL_WARN); - /* If we're pretty sure that we have a compressed directory, and - * we didn't manage to uncompress it, then warn and bail. */ - if (!plausible && !new_body) { - log_fn(LOG_PROTOCOL_WARN, LD_HTTP, - "Unable to decompress HTTP body (server '%s:%d').", - conn->base_.address, conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - if (new_body) { - tor_free(body); - body = new_body; - body_len = new_len; - } + response_handler_args_t args; + memset(&args, 0, sizeof(args)); + args.status_code = status_code; + args.reason = reason; + args.body = body; + args.body_len = body_len; + args.headers = headers; + + switch (conn->base_.purpose) { + case DIR_PURPOSE_FETCH_CONSENSUS: + rv = handle_response_fetch_consensus(conn, &args); + break; + case DIR_PURPOSE_FETCH_CERTIFICATE: + rv = handle_response_fetch_certificate(conn, &args); + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + rv = handle_response_fetch_status_vote(conn, &args); + break; + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + rv = handle_response_fetch_detached_signatures(conn, &args); + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + rv = handle_response_fetch_desc(conn, &args); + break; + case DIR_PURPOSE_FETCH_MICRODESC: + rv = handle_response_fetch_microdesc(conn, &args); + break; + case DIR_PURPOSE_FETCH_RENDDESC_V2: + rv = handle_response_fetch_renddesc_v2(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_DIR: + rv = handle_response_upload_dir(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_SIGNATURES: + rv = handle_response_upload_signatures(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_VOTE: + rv = handle_response_upload_vote(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + rv = handle_response_upload_renddesc_v2(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_HSDESC: + rv = handle_response_upload_hsdesc(conn, &args); + break; + case DIR_PURPOSE_FETCH_HSDESC: + rv = handle_response_fetch_hsdesc_v3(conn, &args); + break; + default: + tor_assert_nonfatal_unreached(); + rv = -1; + break; } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - int r; - const char *flavname = conn->requested_resource; - if (status_code != 200) { - int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; - tor_log(severity, LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching consensus directory.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - networkstatus_consensus_download_failed(status_code, flavname); - return -1; + done: + tor_free(body); + tor_free(headers); + tor_free(reason); + return rv; +} + +/** + * Handler function: processes a response to a request for a networkstatus + * consensus document by checking the consensus, storing it, and marking + * router requests as reachable. + **/ +STATIC int +handle_response_fetch_consensus(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS); + const int status_code = args->status_code; + const char *body = args->body; + const size_t body_len = args->body_len; + const char *reason = args->reason; + const time_t now = approx_time(); + + const char *consensus; + char *new_consensus = NULL; + const char *sourcename; + + int r; + const char *flavname = conn->requested_resource; + if (status_code != 200) { + int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; + tor_log(severity, LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching consensus directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + networkstatus_consensus_download_failed(status_code, flavname); + return -1; + } + + if (looks_like_a_consensus_diff(body, body_len)) { + /* First find our previous consensus. Maybe it's in ram, maybe not. */ + cached_dir_t *cd = dirserv_get_consensus(flavname); + const char *consensus_body; + char *owned_consensus = NULL; + if (cd) { + consensus_body = cd->dir; + } else { + owned_consensus = networkstatus_read_cached_consensus(flavname); + consensus_body = owned_consensus; } - log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if ((r=networkstatus_set_current_consensus(body, flavname, 0, - conn->identity_digest))<0) { - log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, - "Unable to load %s consensus directory downloaded from " - "server '%s:%d'. I'll try again soon.", - flavname, conn->base_.address, conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); + if (!consensus_body) { + log_warn(LD_DIR, "Received a consensus diff, but we can't find " + "any %s-flavored consensus in our current cache.",flavname); networkstatus_consensus_download_failed(0, flavname); + // XXXX if this happens too much, see below return -1; } - /* If we launched other fetches for this consensus, cancel them. */ - connection_dir_close_consensus_fetches(conn, flavname); - - /* launches router downloads as needed */ - routers_update_all_from_networkstatus(now, 3); - update_microdescs_from_networkstatus(now); - update_microdesc_downloads(now); - directory_info_has_arrived(now, 0, 0); - if (authdir_mode_v3(get_options())) { - sr_act_post_consensus( - networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); - } - log_info(LD_DIR, "Successfully loaded consensus."); - } - - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) { - if (status_code != 200) { - log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/keys/%s\".", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource); - connection_dir_download_cert_failed(conn, status_code); - tor_free(body); tor_free(headers); tor_free(reason); + new_consensus = consensus_diff_apply(consensus_body, body); + tor_free(owned_consensus); + if (new_consensus == NULL) { + log_warn(LD_DIR, "Could not apply consensus diff received from server " + "'%s:%d'", conn->base_.address, conn->base_.port); + // XXXX If this happens too many times, we should maybe not use + // XXXX this directory for diffs any more? + networkstatus_consensus_download_failed(0, flavname); return -1; } - log_info(LD_DIR,"Received authority certificates (size %d) from server " + log_info(LD_DIR, "Applied consensus diff (size %d) from server " + "'%s:%d', resulting in a new consensus document (size %d).", + (int)body_len, conn->base_.address, conn->base_.port, + (int)strlen(new_consensus)); + consensus = new_consensus; + sourcename = "generated based on a diff"; + } else { + log_info(LD_DIR,"Received consensus directory (body size %d) from server " "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); + consensus = body; + sourcename = "downloaded"; + } - /* - * Tell trusted_dirs_load_certs_from_string() whether it was by fp - * or fp-sk pair. - */ - src_code = -1; - if (!strcmpstart(conn->requested_resource, "fp/")) { - src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST; - } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) { - src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST; - } + if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + conn->identity_digest))<0) { + log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, + "Unable to load %s consensus directory %s from " + "server '%s:%d'. I'll try again soon.", + flavname, sourcename, conn->base_.address, conn->base_.port); + networkstatus_consensus_download_failed(0, flavname); + tor_free(new_consensus); + return -1; + } - if (src_code != -1) { - if (trusted_dirs_load_certs_from_string(body, src_code, 1, - conn->identity_digest)<0) { - log_warn(LD_DIR, "Unable to parse fetched certificates"); - /* if we fetched more than one and only some failed, the successful - * ones got flushed to disk so it's safe to call this on them */ - connection_dir_download_cert_failed(conn, status_code); - } else { - directory_info_has_arrived(now, 0, 0); - log_info(LD_DIR, "Successfully loaded certificates from fetch."); - } - } else { - log_warn(LD_DIR, - "Couldn't figure out what to do with fetched certificates for " - "unknown resource %s", - conn->requested_resource); + /* If we launched other fetches for this consensus, cancel them. */ + connection_dir_close_consensus_fetches(conn, flavname); + + /* update the list of routers and directory guards */ + routers_update_all_from_networkstatus(now, 3); + update_microdescs_from_networkstatus(now); + directory_info_has_arrived(now, 0, 0); + + if (authdir_mode_v3(get_options())) { + sr_act_post_consensus( + networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); + } + log_info(LD_DIR, "Successfully loaded consensus."); + + tor_free(new_consensus); + return 0; +} + +/** + * Handler function: processes a response to a request for one or more + * authority certificates + **/ +static int +handle_response_fetch_certificate(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + if (status_code != 200) { + log_warn(LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/keys/%s\".", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); + return -1; + } + log_info(LD_DIR,"Received authority certificates (body size %d) from " + "server '%s:%d'", + (int)body_len, conn->base_.address, conn->base_.port); + + /* + * Tell trusted_dirs_load_certs_from_string() whether it was by fp + * or fp-sk pair. + */ + int src_code = -1; + if (!strcmpstart(conn->requested_resource, "fp/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST; + } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST; + } + + if (src_code != -1) { + if (trusted_dirs_load_certs_from_string(body, src_code, 1, + conn->identity_digest)<0) { + log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); + } else { + time_t now = approx_time(); + directory_info_has_arrived(now, 0, 0); + log_info(LD_DIR, "Successfully loaded certificates from fetch."); } + } else { + log_warn(LD_DIR, + "Couldn't figure out what to do with fetched certificates for " + "unknown resource %s", + conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) { - const char *msg; - int st; - log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - log_warn(LD_DIR, + return 0; +} + +/** + * Handler function: processes a response to a request for an authority's + * current networkstatus vote. + **/ +static int +handle_response_fetch_status_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg; + int st; + log_info(LD_DIR,"Got votes (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, "Received http status code %d (%s) from server " "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".", status_code, escaped(reason), conn->base_.address, conn->base_.port, conn->requested_resource); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - dirvote_add_vote(body, &msg, &st); - if (st > 299) { - log_warn(LD_DIR, "Error adding retrieved vote: %s", msg); - } else { - log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg); - } + return -1; } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { - const char *msg = NULL; - log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - log_warn(LD_DIR, + dirvote_add_vote(body, &msg, &st); + if (st > 299) { + log_warn(LD_DIR, "Error adding retrieved vote: %s", msg); + } else { + log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg); + } + + return 0; +} + +/** + * Handler function: processes a response to a request for the signatures + * that an authority knows about on a given consensus. + **/ +static int +handle_response_fetch_detached_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg = NULL; + log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, "Received http status code %d (%s) from server '%s:%d' while fetching " "\"/tor/status-vote/next/consensus-signatures.z\".", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } - if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) { - log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s", - conn->base_.address, conn->base_.port, msg?msg:"???"); - } + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + return -1; + } + if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) { + log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s", + conn->base_.address, conn->base_.port, msg?msg:"???"); } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || - conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { - int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO; - smartlist_t *which = NULL; - int n_asked_for = 0; - int descriptor_digests = conn->requested_resource && - !strcmpstart(conn->requested_resource,"d/"); - log_info(LD_DIR,"Received %s (size %d) from server '%s:%d'", - was_ei ? "extra server info" : "server info", - (int)body_len, conn->base_.address, conn->base_.port); - if (conn->requested_resource && - (!strcmpstart(conn->requested_resource,"d/") || - !strcmpstart(conn->requested_resource,"fp/"))) { - which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource + - (descriptor_digests ? 2 : 3), - which, NULL, 0); - n_asked_for = smartlist_len(which); - } - if (status_code != 200) { - int dir_okay = status_code == 404 || - (status_code == 400 && !strcmp(reason, "Servers unavailable.")); - /* 404 means that it didn't have them; no big deal. - * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */ - log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR, - "Received http status code %d (%s) from server '%s:%d' " - "while fetching \"/tor/server/%s\". I'll try again soon.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource); - if (!which) { - connection_dir_download_routerdesc_failed(conn); - } else { - dir_routerdesc_download_failed(which, status_code, - conn->router_purpose, - was_ei, descriptor_digests); - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - } - tor_free(body); tor_free(headers); tor_free(reason); - return dir_okay ? 0 : -1; - } - /* Learn the routers, assuming we requested by fingerprint or "all" - * or "authority". - * - * We use "authority" to fetch our own descriptor for - * testing, and to fetch bridge descriptors for bootstrapping. Ignore - * the output of "authority" requests unless we are using bridges, - * since otherwise they'll be the response from reachability tests, - * and we don't really want to add that to our routerlist. */ - if (which || (conn->requested_resource && - (!strcmpstart(conn->requested_resource, "all") || - (!strcmpstart(conn->requested_resource, "authority") && - get_options()->UseBridges)))) { - /* as we learn from them, we remove them from 'which' */ - if (was_ei) { - router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which, - descriptor_digests); - } else { - //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, - // descriptor_digests, conn->router_purpose); - if (load_downloaded_routers(body, which, descriptor_digests, - conn->router_purpose, - conn->base_.address)) - directory_info_has_arrived(now, 0, 0); - } - } - if (which) { /* mark remaining ones as failed */ - log_info(LD_DIR, "Received %d/%d %s requested from %s:%d", - n_asked_for-smartlist_len(which), n_asked_for, - was_ei ? "extra-info documents" : "router descriptors", - conn->base_.address, (int)conn->base_.port); - if (smartlist_len(which)) { - dir_routerdesc_download_failed(which, status_code, - conn->router_purpose, - was_ei, descriptor_digests); - } - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - } - if (directory_conn_is_self_reachability_test(conn)) - router_dirport_found_reachable(); - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) { - smartlist_t *which = NULL; - log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " - "size %d) from server '%s:%d'", - status_code, (int)body_len, conn->base_.address, - conn->base_.port); - tor_assert(conn->requested_resource && - !strcmpstart(conn->requested_resource, "d/")); + return 0; +} + +/** + * Handler function: processes a response to a request for a group of server + * descriptors or an extrainfo documents. + **/ +static int +handle_response_fetch_desc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO; + smartlist_t *which = NULL; + int n_asked_for = 0; + int descriptor_digests = conn->requested_resource && + !strcmpstart(conn->requested_resource,"d/"); + log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'", + was_ei ? "extra server info" : "server info", + (int)body_len, conn->base_.address, conn->base_.port); + if (conn->requested_resource && + (!strcmpstart(conn->requested_resource,"d/") || + !strcmpstart(conn->requested_resource,"fp/"))) { which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+2, - which, NULL, - DSR_DIGEST256|DSR_BASE64); - if (status_code != 200) { - log_info(LD_DIR, "Received status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " - "soon.", - status_code, escaped(reason), conn->base_.address, - (int)conn->base_.port, conn->requested_resource); - dir_microdesc_download_failed(which, status_code); + dir_split_resource_into_fingerprints(conn->requested_resource + + (descriptor_digests ? 2 : 3), + which, NULL, 0); + n_asked_for = smartlist_len(which); + } + if (status_code != 200) { + int dir_okay = status_code == 404 || + (status_code == 400 && !strcmp(reason, "Servers unavailable.")); + /* 404 means that it didn't have them; no big deal. + * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */ + log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR, + "Received http status code %d (%s) from server '%s:%d' " + "while fetching \"/tor/server/%s\". I'll try again soon.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + if (!which) { + connection_dir_download_routerdesc_failed(conn); + } else { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); smartlist_free(which); - tor_free(body); tor_free(headers); tor_free(reason); - return 0; + } + return dir_okay ? 0 : -1; + } + /* Learn the routers, assuming we requested by fingerprint or "all" + * or "authority". + * + * We use "authority" to fetch our own descriptor for + * testing, and to fetch bridge descriptors for bootstrapping. Ignore + * the output of "authority" requests unless we are using bridges, + * since otherwise they'll be the response from reachability tests, + * and we don't really want to add that to our routerlist. */ + if (which || (conn->requested_resource && + (!strcmpstart(conn->requested_resource, "all") || + (!strcmpstart(conn->requested_resource, "authority") && + get_options()->UseBridges)))) { + /* as we learn from them, we remove them from 'which' */ + if (was_ei) { + router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which, + descriptor_digests); } else { - smartlist_t *mds; - mds = microdescs_add_to_cache(get_microdesc_cache(), - body, body+body_len, SAVED_NOWHERE, 0, - now, which); - if (smartlist_len(which)) { - /* Mark remaining ones as failed. */ - dir_microdesc_download_failed(which, status_code); - } - if (mds && smartlist_len(mds)) { - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, + // descriptor_digests, conn->router_purpose); + if (load_downloaded_routers(body, which, descriptor_digests, + conn->router_purpose, + conn->base_.address)) { + time_t now = approx_time(); directory_info_has_arrived(now, 0, 1); } - SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); - smartlist_free(which); - smartlist_free(mds); } } + if (which) { /* mark remaining ones as failed */ + log_info(LD_DIR, "Received %d/%d %s requested from %s:%d", + n_asked_for-smartlist_len(which), n_asked_for, + was_ei ? "extra-info documents" : "router descriptors", + conn->base_.address, (int)conn->base_.port); + if (smartlist_len(which)) { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); + } + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + } + if (directory_conn_is_self_reachability_test(conn)) + router_dirport_found_reachable(); - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR) { - switch (status_code) { - case 200: { - dir_server_t *ds = - router_get_trusteddirserver_by_digest(conn->identity_digest); - char *rejected_hdr = http_get_header(headers, - "X-Descriptor-Not-New: "); - if (rejected_hdr) { - if (!strcmp(rejected_hdr, "Yes")) { - log_info(LD_GENERAL, - "Authority '%s' declined our descriptor (not new)", - ds->nickname); - /* XXXX use this information; be sure to upload next one - * sooner. -NM */ - /* XXXX++ On further thought, the task above implies that we're - * basing our regenerate-descriptor time on when we uploaded the - * last descriptor, not on the published time of the last - * descriptor. If those are different, that's a bad thing to - * do. -NM */ - } - tor_free(rejected_hdr); - } - log_info(LD_GENERAL,"eof (status 200) after uploading server " - "descriptor: finished."); - control_event_server_status( - LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d", - conn->base_.address, conn->base_.port); - - ds->has_accepted_serverdesc = 1; - if (directories_have_accepted_server_descriptor()) - control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR"); - } - break; - case 400: - log_warn(LD_GENERAL,"http status 400 (%s) response from " - "dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - control_event_server_status(LOG_WARN, - "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"", - conn->base_.address, conn->base_.port, escaped(reason)); - break; - default: - log_warn(LD_GENERAL, - "http status %d (%s) reason unexpected while uploading " - "descriptor to server '%s:%d').", + return 0; +} + +/** + * Handler function: processes a response to a request for a group of + * microdescriptors + **/ +STATIC int +handle_response_fetch_microdesc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + smartlist_t *which = NULL; + log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " + "body size %d) from server '%s:%d'", + status_code, (int)body_len, conn->base_.address, + conn->base_.port); + tor_assert(conn->requested_resource && + !strcmpstart(conn->requested_resource, "d/")); + tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN)); + which = smartlist_new(); + dir_split_resource_into_fingerprints(conn->requested_resource+2, + which, NULL, + DSR_DIGEST256|DSR_BASE64); + if (status_code != 200) { + log_info(LD_DIR, "Received status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " + "soon.", status_code, escaped(reason), conn->base_.address, - conn->base_.port); - break; + (int)conn->base_.port, conn->requested_resource); + dir_microdesc_download_failed(which, status_code, conn->identity_digest); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + return 0; + } else { + smartlist_t *mds; + time_t now = approx_time(); + mds = microdescs_add_to_cache(get_microdesc_cache(), + body, body+body_len, SAVED_NOWHERE, 0, + now, which); + if (smartlist_len(which)) { + /* Mark remaining ones as failed. */ + dir_microdesc_download_failed(which, status_code, conn->identity_digest); + } + if (mds && smartlist_len(mds)) { + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); + directory_info_has_arrived(now, 0, 1); } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + smartlist_free(mds); } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE) { - switch (status_code) { - case 200: { - log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d", + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload our + * router descriptor. + **/ +static int +handle_response_upload_dir(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *headers = args->headers; + + switch (status_code) { + case 200: { + dir_server_t *ds = + router_get_trusteddirserver_by_digest(conn->identity_digest); + char *rejected_hdr = http_get_header(headers, + "X-Descriptor-Not-New: "); + if (rejected_hdr) { + if (!strcmp(rejected_hdr, "Yes")) { + log_info(LD_GENERAL, + "Authority '%s' declined our descriptor (not new)", + ds->nickname); + /* XXXX use this information; be sure to upload next one + * sooner. -NM */ + /* XXXX++ On further thought, the task above implies that we're + * basing our regenerate-descriptor time on when we uploaded the + * last descriptor, not on the published time of the last + * descriptor. If those are different, that's a bad thing to + * do. -NM */ + } + tor_free(rejected_hdr); + } + log_info(LD_GENERAL,"eof (status 200) after uploading server " + "descriptor: finished."); + control_event_server_status( + LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d", conn->base_.address, conn->base_.port); - } - break; - case 400: - log_warn(LD_DIR,"http status 400 (%s) response after uploading " - "vote to dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - break; - default: - log_warn(LD_GENERAL, - "http status %d (%s) reason unexpected while uploading " - "vote to server '%s:%d').", + + ds->has_accepted_serverdesc = 1; + if (directories_have_accepted_server_descriptor()) + control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR"); + } + break; + case 400: + log_warn(LD_GENERAL,"http status 400 (%s) response from " + "dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + control_event_server_status(LOG_WARN, + "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"", + conn->base_.address, conn->base_.port, escaped(reason)); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "descriptor to server '%s:%d'. Possibly the server is " + "misconfigured?", status_code, escaped(reason), conn->base_.address, conn->base_.port); - break; - } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ + break; } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES) { - switch (status_code) { - case 200: { - log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d", - conn->base_.address, conn->base_.port); - } - break; - case 400: - log_warn(LD_DIR,"http status 400 (%s) response after uploading " - "signatures to dirserver '%s:%d'. Please correct.", - escaped(reason), conn->base_.address, conn->base_.port); - break; - default: - log_warn(LD_GENERAL, - "http status %d (%s) reason unexpected while uploading " - "signatures to server '%s:%d').", + return 0; +} + +/** + * Handler function: processes a response to POST request to upload our + * own networkstatus vote. + **/ +static int +handle_response_upload_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "vote to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "vote to server '%s:%d'.", status_code, escaped(reason), conn->base_.address, conn->base_.port); - break; + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + return 0; +} + +/** + * Handler function: processes a response to POST request to upload our + * view of the signatures on the current consensus. + **/ +static int +handle_response_upload_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "signatures to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "signatures to server '%s:%d'.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + + return 0; +} + +/** + * Handler function: processes a response to a request for a v3 hidden service + * descriptor. + **/ +STATIC int +handle_response_fetch_hsdesc_v3(dir_connection_t *conn, + const response_handler_args_t *args) +{ + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + tor_assert(conn->hs_ident); + + log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))", + (int)body_len, status_code, escaped(reason)); + + switch (status_code) { + case 200: + /* We got something: Try storing it in the cache. */ + if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) { + log_info(LD_REND, "Failed to store hidden service descriptor"); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "BAD_DESC"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + } else { + log_info(LD_REND, "Stored hidden service descriptor successfully."); + TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC; + hs_client_desc_has_arrived(conn->hs_ident); + /* Fire control port RECEIVED event. */ + hs_control_desc_event_received(conn->hs_ident, conn->identity_digest); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + body); } - /* return 0 in all cases, since we don't want to mark any - * dirservers down just because they don't like us. */ - } - - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { - #define SEND_HS_DESC_FAILED_EVENT(reason) ( \ - control_event_hs_descriptor_failed(conn->rend_data, \ - conn->identity_digest, \ - reason) ) - #define SEND_HS_DESC_FAILED_CONTENT() ( \ - control_event_hs_descriptor_content(conn->rend_data->onion_address, \ - conn->requested_resource, \ - conn->identity_digest, \ - NULL) ) - tor_assert(conn->rend_data); - log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " - "(%s))", - (int)body_len, status_code, escaped(reason)); - switch (status_code) { - case 200: - { - rend_cache_entry_t *entry = NULL; - - if (rend_cache_store_v2_desc_as_client(body, - conn->requested_resource, conn->rend_data, &entry) < 0) { - log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " - "Retrying at another directory."); - /* We'll retry when connection_about_to_close_connection() - * cleans this dir conn up. */ - SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); - SEND_HS_DESC_FAILED_CONTENT(); - } else { - char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; - /* Should never be NULL here if we found the descriptor. */ - tor_assert(entry); - rend_get_service_id(entry->parsed->pk, service_id); - - /* success. notify pending connections about this. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor."); - control_event_hs_descriptor_received(service_id, + break; + case 404: + /* Not there. We'll retry when connection_about_to_close_connection() + * tries to clean this conn up. */ + log_info(LD_REND, "Fetching hidden service v3 descriptor not found: " + "Retrying at another directory."); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "NOT_FOUND"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + break; + case 400: + log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: " + "http status 400 (%s). Dirserver didn't like our " + "query? Retrying at another directory.", + escaped(reason)); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "QUERY_REJECTED"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + break; + default: + log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: " + "http status %d (%s) response unexpected from HSDir server " + "'%s:%d'. Retrying at another directory.", + status_code, escaped(reason), TO_CONN(conn)->address, + TO_CONN(conn)->port); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "UNEXPECTED"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + break; + } + + return 0; +} + +/** + * Handler function: processes a response to a request for a v2 hidden service + * descriptor. + **/ +static int +handle_response_fetch_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + +#define SEND_HS_DESC_FAILED_EVENT(reason) \ + (control_event_hsv2_descriptor_failed(conn->rend_data, \ + conn->identity_digest, \ + reason)) +#define SEND_HS_DESC_FAILED_CONTENT() \ + (control_event_hs_descriptor_content( \ + rend_data_get_address(conn->rend_data), \ + conn->requested_resource, \ + conn->identity_digest, \ + NULL)) + + tor_assert(conn->rend_data); + log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d " + "(%s))", + (int)body_len, status_code, escaped(reason)); + switch (status_code) { + case 200: + { + rend_cache_entry_t *entry = NULL; + + if (rend_cache_store_v2_desc_as_client(body, + conn->requested_resource, + conn->rend_data, &entry) < 0) { + log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " + "Retrying at another directory."); + /* We'll retry when connection_about_to_close_connection() + * cleans this dir conn up. */ + SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); + SEND_HS_DESC_FAILED_CONTENT(); + } else { + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + /* Should never be NULL here if we found the descriptor. */ + tor_assert(entry); + rend_get_service_id(entry->parsed->pk, service_id); + + /* success. notify pending connections about this. */ + log_info(LD_REND, "Successfully fetched v2 rendezvous " + "descriptor."); + control_event_hsv2_descriptor_received(service_id, conn->rend_data, conn->identity_digest); - control_event_hs_descriptor_content(service_id, - conn->requested_resource, - conn->identity_digest, - body); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; - rend_client_desc_trynow(service_id); - memwipe(service_id, 0, sizeof(service_id)); - } - break; + control_event_hs_descriptor_content(service_id, + conn->requested_resource, + conn->identity_digest, + body); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; + rend_client_desc_trynow(service_id); + memwipe(service_id, 0, sizeof(service_id)); } - case 404: - /* Not there. We'll retry when - * connection_about_to_close_connection() cleans this conn up. */ - log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " - "Retrying at another directory."); - SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); - SEND_HS_DESC_FAILED_CONTENT(); - break; - case 400: - log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " - "http status 400 (%s). Dirserver didn't like our " - "v2 rendezvous query? Retrying at another directory.", - escaped(reason)); - SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); - SEND_HS_DESC_FAILED_CONTENT(); - break; - default: - log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " - "http status %d (%s) response unexpected while " - "fetching v2 hidden service descriptor (server '%s:%d'). " - "Retrying at another directory.", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); - SEND_HS_DESC_FAILED_CONTENT(); - break; + break; } + case 404: + /* Not there. We'll retry when + * connection_about_to_close_connection() cleans this conn up. */ + log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " + "Retrying at another directory."); + SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + case 400: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status 400 (%s). Dirserver didn't like our " + "v2 rendezvous query? Retrying at another directory.", + escaped(reason)); + SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + default: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status %d (%s) response unexpected while " + "fetching v2 hidden service descriptor (server '%s:%d'). " + "Retrying at another directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { - #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \ - control_event_hs_descriptor_upload_failed( \ - conn->identity_digest, \ - conn->rend_data->onion_address, \ - reason) ) - log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " - "(%s))", - status_code, escaped(reason)); - /* Without the rend data, we'll have a problem identifying what has been - * uploaded for which service. */ - tor_assert(conn->rend_data); - switch (status_code) { - case 200: - log_info(LD_REND, - "Uploading rendezvous descriptor: finished with status " - "200 (%s)", escaped(reason)); - control_event_hs_descriptor_uploaded(conn->identity_digest, - conn->rend_data->onion_address); - rend_service_desc_has_uploaded(conn->rend_data); - break; - case 400: - log_warn(LD_REND,"http status 400 (%s) response from dirserver " - "'%s:%d'. Malformed rendezvous descriptor?", - escaped(reason), conn->base_.address, conn->base_.port); - SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); - break; - default: - log_warn(LD_REND,"http status %d (%s) response unexpected (server " - "'%s:%d').", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); - break; - } + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload a v2 + * hidden service descriptor. + **/ +static int +handle_response_upload_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + +#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) \ + (control_event_hs_descriptor_upload_failed( \ + conn->identity_digest, \ + rend_data_get_address(conn->rend_data), \ + reason)) + + log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " + "(%s))", + status_code, escaped(reason)); + /* Without the rend data, we'll have a problem identifying what has been + * uploaded for which service. */ + tor_assert(conn->rend_data); + switch (status_code) { + case 200: + log_info(LD_REND, + "Uploading rendezvous descriptor: finished with status " + "200 (%s)", escaped(reason)); + control_event_hs_descriptor_uploaded(conn->identity_digest, + rend_data_get_address(conn->rend_data)); + rend_service_desc_has_uploaded(conn->rend_data); + break; + case 400: + log_warn(LD_REND,"http status 400 (%s) response from dirserver " + "'%s:%d'. Malformed rendezvous descriptor?", + escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); + break; + default: + log_warn(LD_REND,"http status %d (%s) response unexpected (server " + "'%s:%d').", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); + break; } - tor_free(body); tor_free(headers); tor_free(reason); + + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload an + * hidden service descriptor. + **/ +static int +handle_response_upload_hsdesc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + const int status_code = args->status_code; + const char *reason = args->reason; + + tor_assert(conn); + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_HSDESC); + + log_info(LD_REND, "Uploaded hidden service descriptor (status %d " + "(%s))", + status_code, escaped(reason)); + /* For this directory response, it MUST have an hidden service identifier on + * this connection. */ + tor_assert(conn->hs_ident); + switch (status_code) { + case 200: + log_info(LD_REND, "Uploading hidden service descriptor: " + "finished with status 200 (%s)", escaped(reason)); + hs_control_desc_event_uploaded(conn->hs_ident, conn->identity_digest); + break; + case 400: + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Uploading hidden service descriptor: http " + "status 400 (%s) response from dirserver " + "'%s:%d'. Malformed hidden service descriptor?", + escaped(reason), conn->base_.address, conn->base_.port); + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "UPLOAD_REJECTED"); + break; + default: + log_warn(LD_REND, "Uploading hidden service descriptor: http " + "status %d (%s) response unexpected (server " + "'%s:%d').", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "UNEXPECTED"); + break; + } + return 0; } @@ -2524,6 +3413,33 @@ connection_dir_process_inbuf(dir_connection_t *conn) return 0; } +/** We are closing a dir connection: If <b>dir_conn</b> is a dir connection + * that tried to fetch an HS descriptor, check if it successfully fetched it, + * or if we need to try again. */ +static void +refetch_hsdesc_if_needed(dir_connection_t *dir_conn) +{ + connection_t *conn = TO_CONN(dir_conn); + + /* If we were trying to fetch a v2 rend desc and did not succeed, retry as + * needed. (If a fetch is successful, the connection state is changed to + * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to + * mark that refetching is unnecessary.) */ + if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 && + dir_conn->rend_data && + rend_valid_v2_service_id( + rend_data_get_address(dir_conn->rend_data))) { + rend_client_refetch_v2_renddesc(dir_conn->rend_data); + } + + /* Check for v3 rend desc fetch */ + if (conn->purpose == DIR_PURPOSE_FETCH_HSDESC && + dir_conn->hs_ident && + !ed25519_public_key_is_zero(&dir_conn->hs_ident->identity_pk)) { + hs_client_refetch_hsdesc(&dir_conn->hs_ident->identity_pk); + } +} + /** Called when we're about to finally unlink and free a directory connection: * perform necessary accounting and cleanup */ void @@ -2536,31 +3452,39 @@ connection_dir_about_to_close(dir_connection_t *dir_conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - /* If we were trying to fetch a v2 rend desc and did not succeed, - * retry as needed. (If a fetch is successful, the connection state - * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 to mark that - * refetching is unnecessary.) */ - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 && - dir_conn->rend_data && - strlen(dir_conn->rend_data->onion_address) == REND_SERVICE_ID_LEN_BASE32) - rend_client_refetch_v2_renddesc(dir_conn->rend_data); + + refetch_hsdesc_if_needed(dir_conn); } /** Create an http response for the client <b>conn</b> out of * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>. */ static void -write_http_status_line(dir_connection_t *conn, int status, +write_short_http_response(dir_connection_t *conn, int status, const char *reason_phrase) { - char buf[256]; - if (tor_snprintf(buf, sizeof(buf), "HTTP/1.0 %d %s\r\n\r\n", - status, reason_phrase ? reason_phrase : "OK") < 0) { - log_warn(LD_BUG,"status line too long."); - return; + char *buf = NULL; + char *datestring = NULL; + + IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */ + reason_phrase = "unspecified"; + } + + if (server_mode(get_options())) { + /* include the Date: header, but only if we're a relay or bridge */ + char datebuf[RFC1123_TIME_LEN+1]; + format_rfc1123_time(datebuf, time(NULL)); + tor_asprintf(&datestring, "Date: %s\r\n", datebuf); } + + tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n", + status, reason_phrase, datestring?datestring:""); + log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); - connection_write_to_buf(buf, strlen(buf), TO_CONN(conn)); + connection_buf_add(buf, strlen(buf), TO_CONN(conn)); + + tor_free(datestring); + tor_free(buf); } /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf, @@ -2577,76 +3501,160 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, long cache_lifetime) { char date[RFC1123_TIME_LEN+1]; - char tmp[1024]; - char *cp; time_t now = time(NULL); + buf_t *buf = buf_new_with_capacity(1024); tor_assert(conn); format_rfc1123_time(date, now); - cp = tmp; - tor_snprintf(cp, sizeof(tmp), - "HTTP/1.0 200 OK\r\nDate: %s\r\n", - date); - cp += strlen(tmp); + + buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date); if (type) { - tor_snprintf(cp, sizeof(tmp)-(cp-tmp), "Content-Type: %s\r\n", type); - cp += strlen(cp); + buf_add_printf(buf, "Content-Type: %s\r\n", type); } if (!is_local_addr(&conn->base_.addr)) { /* Don't report the source address for a nearby/private connection. * Otherwise we tend to mis-report in cases where incoming ports are * being forwarded to a Tor server running behind the firewall. */ - tor_snprintf(cp, sizeof(tmp)-(cp-tmp), - X_ADDRESS_HEADER "%s\r\n", conn->base_.address); - cp += strlen(cp); + buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address); } if (encoding) { - tor_snprintf(cp, sizeof(tmp)-(cp-tmp), - "Content-Encoding: %s\r\n", encoding); - cp += strlen(cp); + buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding); } if (length >= 0) { - tor_snprintf(cp, sizeof(tmp)-(cp-tmp), - "Content-Length: %ld\r\n", (long)length); - cp += strlen(cp); + buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length); } if (cache_lifetime > 0) { char expbuf[RFC1123_TIME_LEN+1]; format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime)); /* We could say 'Cache-control: max-age=%d' here if we start doing * http/1.1 */ - tor_snprintf(cp, sizeof(tmp)-(cp-tmp), - "Expires: %s\r\n", expbuf); - cp += strlen(cp); + buf_add_printf(buf, "Expires: %s\r\n", expbuf); } else if (cache_lifetime == 0) { /* We could say 'Cache-control: no-cache' here if we start doing * http/1.1 */ - strlcpy(cp, "Pragma: no-cache\r\n", sizeof(tmp)-(cp-tmp)); - cp += strlen(cp); + buf_add_string(buf, "Pragma: no-cache\r\n"); } if (extra_headers) { - strlcpy(cp, extra_headers, sizeof(tmp)-(cp-tmp)); - cp += strlen(cp); + buf_add_string(buf, extra_headers); } - if (sizeof(tmp)-(cp-tmp) > 3) - memcpy(cp, "\r\n", 3); - else - tor_assert(0); - connection_write_to_buf(tmp, strlen(tmp), TO_CONN(conn)); + buf_add_string(buf, "\r\n"); + + connection_buf_add_buf(TO_CONN(conn), buf); + buf_free(buf); } /** As write_http_response_header_impl, but sets encoding and content-typed * based on whether the response will be <b>compressed</b> or not. */ static void -write_http_response_header(dir_connection_t *conn, ssize_t length, - int compressed, long cache_lifetime) +write_http_response_headers(dir_connection_t *conn, ssize_t length, + compress_method_t method, + const char *extra_headers, long cache_lifetime) { + const char *methodname = compression_method_get_name(method); + const char *doctype; + if (method == NO_METHOD) + doctype = "text/plain"; + else + doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - compressed?"application/octet-stream":"text/plain", - compressed?"deflate":"identity", - NULL, - cache_lifetime); + doctype, + methodname, + extra_headers, + cache_lifetime); +} + +/** As write_http_response_headers, but assumes extra_headers is NULL */ +static void +write_http_response_header(dir_connection_t *conn, ssize_t length, + compress_method_t method, + long cache_lifetime) +{ + write_http_response_headers(conn, length, method, NULL, cache_lifetime); +} + +/** Array of compression methods to use (if supported) for serving + * precompressed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_precompressed[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Array of compression methods to use (if supported) for serving + * streamed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_streaming_compression[] = { + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Array of allowed compression methods to use (if supported) when receiving a + * response from a request that was required to be anonymous. */ +static compress_method_t client_meth_allowed_anonymous_compression[] = { + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>, + * and convert them to a bitfield where compression method x is supported if + * and only if 1 << x is set in the bitfield. */ +STATIC unsigned +parse_accept_encoding_header(const char *h) +{ + unsigned result = (1u << NO_METHOD); + smartlist_t *methods = smartlist_new(); + smartlist_split_string(methods, h, ",", + SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + SMARTLIST_FOREACH_BEGIN(methods, const char *, m) { + compress_method_t method = compression_method_get_by_name(m); + if (method != UNKNOWN_METHOD) { + tor_assert(((unsigned)method) < 8*sizeof(unsigned)); + result |= (1u << method); + } + } SMARTLIST_FOREACH_END(m); + SMARTLIST_FOREACH_BEGIN(methods, char *, m) { + tor_free(m); + } SMARTLIST_FOREACH_END(m); + smartlist_free(methods); + return result; +} + +/** Array of compression methods to use (if supported) for requesting + * compressed data, ordered from best to worst. */ +static compress_method_t client_meth_pref[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Return a newly allocated string containing a comma separated list of + * supported encodings. */ +STATIC char * +accept_encoding_header(void) +{ + smartlist_t *methods = smartlist_new(); + char *header = NULL; + compress_method_t method; + unsigned i; + + for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { + method = client_meth_pref[i]; + if (tor_compress_supports_method(method)) + smartlist_add(methods, (char *)compression_method_get_name(method)); + } + + header = smartlist_join_strings(methods, ", ", 0, NULL); + smartlist_free(methods); + + return header; } /** Decide whether a client would accept the consensus we have. @@ -2664,48 +3672,46 @@ write_http_response_header(dir_connection_t *conn, ssize_t length, * consensus, 0 otherwise. */ int -client_likes_consensus(networkstatus_t *v, const char *want_url) +client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url) { - smartlist_t *want_authorities = smartlist_new(); + smartlist_t *voters = smartlist_new(); int need_at_least; int have = 0; + if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) { + smartlist_free(voters); + return 1; // We don't know the voters; assume the client won't mind. */ + } + + smartlist_t *want_authorities = smartlist_new(); dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { - char want_digest[DIGEST_LEN]; - size_t want_len = strlen(d)/2; - if (want_len > DIGEST_LEN) - want_len = DIGEST_LEN; - - if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) - != (int) want_len) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Failed to decode requested authority digest %s.", escaped(d)); - continue; - }; - SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { - if (smartlist_len(vi->sigs) && - tor_memeq(vi->identity_digest, want_digest, want_len)) { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) { + + SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) { + if (!strcasecmpstart(digest, want_digest)) { have++; break; }; - } SMARTLIST_FOREACH_END(vi); + } SMARTLIST_FOREACH_END(digest); /* early exit, if we already have enough */ if (have >= need_at_least) break; - } SMARTLIST_FOREACH_END(d); + } SMARTLIST_FOREACH_END(want_digest); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); + SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp)); + smartlist_free(voters); return (have >= need_at_least); } /** Return the compression level we should use for sending a compressed * response of size <b>n_bytes</b>. */ -STATIC zlib_compression_level_t +STATIC compression_level_t choose_compression_level(ssize_t n_bytes) { if (! have_been_under_memory_pressure()) { @@ -2723,8 +3729,9 @@ choose_compression_level(ssize_t n_bytes) /** Information passed to handle a GET request. */ typedef struct get_handler_args_t { - /** True if the client asked for compressed data. */ - int compressed; + /** Bitmask of compression methods that the client said (or implied) it + * supported. */ + unsigned compression_supported; /** If nonzero, the time included an if-modified-since header with this * value. */ time_t if_modified_since; @@ -2762,8 +3769,8 @@ static int handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args); static int handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args); -static int handle_get_rendezvous2(dir_connection_t *conn, - const get_handler_args_t *args); +static int handle_get_hs_descriptor_v2(dir_connection_t *conn, + const get_handler_args_t *args); static int handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args); static int handle_get_networkstatus_bridges(dir_connection_t *conn, @@ -2779,7 +3786,8 @@ static const url_table_ent_t url_table[] = { { "/tor/server/", 1, handle_get_descriptor }, { "/tor/extra/", 1, handle_get_descriptor }, { "/tor/keys/", 1, handle_get_keys }, - { "/tor/rendezvous2/", 1, handle_get_rendezvous2 }, + { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 }, + { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 }, { "/tor/robots.txt", 0, handle_get_robots }, { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges }, { NULL, 0, NULL }, @@ -2791,14 +3799,14 @@ static const url_table_ent_t url_table[] = { * conn-\>outbuf. If the request is unrecognized, send a 404. * Return 0 if we handled this successfully, or -1 if we need to close * the connection. */ -STATIC int -directory_handle_command_get(dir_connection_t *conn, const char *headers, - const char *req_body, size_t req_body_len) +MOCK_IMPL(STATIC int, +directory_handle_command_get,(dir_connection_t *conn, const char *headers, + const char *req_body, size_t req_body_len)) { char *url, *url_mem, *header; time_t if_modified_since = 0; - int compressed; - size_t url_len; + int zlib_compressed_in_url; + unsigned compression_methods_supported; /* We ignore the body of a GET request. */ (void)req_body; @@ -2809,7 +3817,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; if (parse_http_url(headers, &url) < 0) { - write_http_status_line(conn, 400, "Bad request"); + write_short_http_response(conn, 400, "Bad request"); return 0; } if ((header = http_get_header(headers, "If-Modified-Since: "))) { @@ -2828,18 +3836,33 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); url_mem = url; - url_len = strlen(url); - compressed = url_len > 2 && !strcmp(url+url_len-2, ".z"); - if (compressed) { - url[url_len-2] = '\0'; - url_len -= 2; + { + size_t url_len = strlen(url); + + zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z"); + if (zlib_compressed_in_url) { + url[url_len-2] = '\0'; + } + } + + if ((header = http_get_header(headers, "Accept-Encoding: "))) { + compression_methods_supported = parse_accept_encoding_header(header); + tor_free(header); + } else { + compression_methods_supported = (1u << NO_METHOD); } + if (zlib_compressed_in_url) { + compression_methods_supported |= (1u << ZLIB_METHOD); + } + + /* Remove all methods that we don't both support. */ + compression_methods_supported &= tor_compress_get_supported_method_bitmask(); get_handler_args_t args; args.url = url; args.headers = headers; args.if_modified_since = if_modified_since; - args.compressed = compressed; + args.compression_supported = compression_methods_supported; int i, result = -1; for (i = 0; url_table[i].string; ++i) { @@ -2856,7 +3879,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } /* we didn't recognize the url */ - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); result = 0; done: @@ -2882,141 +3905,530 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) * this page no matter what.] */ write_http_response_header_impl(conn, dlen, "text/html", "identity", NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); - connection_write_to_buf(frontpage, dlen, TO_CONN(conn)); + connection_buf_add(frontpage, dlen, TO_CONN(conn)); } else { - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); } return 0; } -/** Helper function for GET /tor/status-vote/current/consensus +/** Warn that the cached consensus <b>consensus</b> of type + * <b>flavor</b> is too old and will not be served to clients. Rate-limit the + * warning to avoid logging an entry on every request. */ -static int -handle_get_current_consensus(dir_connection_t *conn, - const get_handler_args_t *args) +static void +warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, + const char *flavor, time_t now) { - const char *url = args->url; - const int compressed = args->compressed; - const time_t if_modified_since = args->if_modified_since; +#define TOO_OLD_WARNING_INTERVAL (60*60) + static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); + char timestamp[ISO_TIME_LEN+1]; + time_t valid_until; + char *dupes; - { - /* v3 network status fetch. */ - smartlist_t *dir_fps = smartlist_new(); - long lifetime = NETWORKSTATUS_CACHE_LIFETIME; - - if (1) { - networkstatus_t *v; - time_t now = time(NULL); - const char *want_fps = NULL; - char *flavor = NULL; - int flav = FLAV_NS; - #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" - /* figure out the flavor if any, and who we wanted to sign the thing */ - if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { - const char *f, *cp; - f = url + strlen(CONSENSUS_FLAVORED_PREFIX); - cp = strchr(f, '/'); - if (cp) { - want_fps = cp+1; - flavor = tor_strndup(f, cp-f); - } else { - flavor = tor_strdup(f); - } - flav = networkstatus_parse_flavor_name(flavor); - if (flav < 0) - flav = FLAV_NS; - } else { - if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) - want_fps = url+strlen(CONSENSUS_URL_PREFIX); - } + if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) + return; - v = networkstatus_get_latest_consensus_by_flavor(flav); + if ((dupes = rate_limit_log(&warned, now))) { + format_local_iso_time(timestamp, valid_until); + log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " + "serve it to clients. It was valid until %s local time and we " + "continued to serve it for up to 24 hours after it expired.%s", + flavor ? flavor : "", flavor ? " " : "", timestamp, dupes); + tor_free(dupes); + } +} - if (v && want_fps && - !client_likes_consensus(v, want_fps)) { - write_http_status_line(conn, 404, "Consensus not signed by sufficient " - "number of requested authorities"); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); - tor_free(flavor); - goto done; - } +/** + * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into + * <b>digest</b>. Return 0 on success. On failure, report that the hash came + * from <b>location</b>, report that we are taking <b>action</b> with it, and + * return -1. + */ +static int +parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location, + const char *action) +{ + if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) == + DIGEST256_LEN) { + return 0; + } else { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "%s contained bogus digest %s; %s.", + location, escaped(hex), action); + return -1; + } +} + +/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>, + * set <b>digest_out<b> to a new smartlist containing every 256-bit + * hex-encoded digest listed in that header and return 0. Otherwise return + * -1. */ +static int +parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) +{ + char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER); + if (hdr == NULL) { + return -1; + } + smartlist_t *hex_digests = smartlist_new(); + *digests_out = smartlist_new(); + smartlist_split_string(hex_digests, hdr, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) { + uint8_t digest[DIGEST256_LEN]; + if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header", + "ignoring")) { + smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest))); + } + } SMARTLIST_FOREACH_END(hex); + SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp)); + smartlist_free(hex_digests); + tor_free(hdr); + return 0; +} + +/** Fallback compression method. The fallback compression method is used in + * case a client requests a non-compressed document. We only store compressed + * documents, so we use this compression method to fetch the document and let + * the spooling system do the streaming decompression. + */ +#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD - { - char *fp = tor_malloc_zero(DIGEST_LEN); - if (flavor) - strlcpy(fp, flavor, DIGEST_LEN); - tor_free(flavor); - smartlist_add(dir_fps, fp); +/** + * Try to find the best consensus diff possible in order to serve a client + * request for a diff from one of the consensuses in <b>digests</b> to the + * current consensus of flavor <b>flav</b>. The client supports the + * compression methods listed in the <b>compression_methods</b> bitfield: + * place the method chosen (if any) into <b>compression_used_out</b>. + */ +static struct consensus_cache_entry_t * +find_best_diff(const smartlist_t *digests, int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + unsigned u; + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + if (0 == (compression_methods & (1u<<method))) + continue; // client doesn't like this one, or we don't have it. + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, + diff_from, DIGEST256_LEN, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; } - lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } + } SMARTLIST_FOREACH_END(diff_from); + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from, + DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + } SMARTLIST_FOREACH_END(diff_from); - if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ - write_http_status_line(conn, 503, "Network status object unavailable"); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); - goto done; + return NULL; +} + +/** Lookup the cached consensus document by the flavor found in <b>flav</b>. + * The preferred set of compression methods should be listed in the + * <b>compression_methods</b> bitfield. The compression method chosen (if any) + * is stored in <b>compression_used_out</b>. */ +static struct consensus_cache_entry_t * +find_best_consensus(int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + unsigned u; + + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + + if (0 == (compression_methods & (1u<<method))) + continue; + + if (consdiffmgr_find_consensus(&result, flav, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; } + } - if (!dirserv_remove_old_statuses(dir_fps, if_modified_since)) { - write_http_status_line(conn, 404, "Not found"); - SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - goto done; - } else if (!smartlist_len(dir_fps)) { - write_http_status_line(conn, 304, "Not modified"); - SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); - goto done; + if (consdiffmgr_find_consensus(&result, flav, + FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + + return NULL; +} + +/** Try to find the best supported compression method possible from a given + * <b>compression_methods</b>. Return NO_METHOD if no mutually supported + * compression method could be found. */ +static compress_method_t +find_best_compression_method(unsigned compression_methods, int stream) +{ + unsigned u; + compress_method_t *methods; + size_t length; + + if (stream) { + methods = srv_meth_pref_streaming_compression; + length = ARRAY_LENGTH(srv_meth_pref_streaming_compression); + } else { + methods = srv_meth_pref_precompressed; + length = ARRAY_LENGTH(srv_meth_pref_precompressed); + } + + for (u = 0; u < length; ++u) { + compress_method_t method = methods[u]; + if (compression_methods & (1u<<method)) + return method; + } + + return NO_METHOD; +} + +/** Check if any of the digests in <b>digests</b> matches the latest consensus + * flavor (given in <b>flavor</b>) that we have available. */ +static int +digest_list_contains_best_consensus(consensus_flavor_t flavor, + const smartlist_t *digests) +{ + const networkstatus_t *ns = NULL; + + if (digests == NULL) + return 0; + + ns = networkstatus_get_latest_consensus_by_flavor(flavor); + + if (ns == NULL) + return 0; + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) { + if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN)) + return 1; + } SMARTLIST_FOREACH_END(digest); + + return 0; +} + +/** Check if the given compression method is allowed for a connection that is + * supposed to be anonymous. Returns 1 if the compression method is allowed, + * otherwise 0. */ +STATIC int +allowed_anonymous_connection_compression_method(compress_method_t method) +{ + unsigned u; + + for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression); + ++u) { + compress_method_t allowed_method = + client_meth_allowed_anonymous_compression[u]; + + if (! tor_compress_supports_method(allowed_method)) + continue; + + if (method == allowed_method) + return 1; + } + + return 0; +} + +/** Log a warning when a remote server has sent us a document using a + * compression method that is not allowed for anonymous directory requests. */ +STATIC void +warn_disallowed_anonymous_compression_method(compress_method_t method) +{ + log_fn(LOG_PROTOCOL_WARN, LD_HTTP, + "Received a %s HTTP response, which is not " + "allowed for anonymous directory requests.", + compression_method_get_human_name(method)); +} + +/** Encodes the results of parsing a consensus request to figure out what + * consensus, and possibly what diffs, the user asked for. */ +typedef struct { + /** name of the flavor to retrieve. */ + char *flavor; + /** flavor to retrive, as enum. */ + consensus_flavor_t flav; + /** plus-separated list of authority fingerprints; see + * client_likes_consensus(). Aliases the URL in the request passed to + * parse_consensus_request(). */ + const char *want_fps; + /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses + * to return a diff from. */ + smartlist_t *diff_from_digests; + /** If true, never send a full consensus. If there is no diff, send + * a 404 instead. */ + int diff_only; +} parsed_consensus_request_t; + +/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since + * it is stack-allocated. */ +static void +parsed_consensus_request_clear(parsed_consensus_request_t *req) +{ + if (!req) + return; + tor_free(req->flavor); + if (req->diff_from_digests) { + SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d)); + smartlist_free(req->diff_from_digests); + } + memset(req, 0, sizeof(parsed_consensus_request_t)); +} + +/** + * Parse the URL and relevant headers of <b>args</b> for a current-consensus + * request to learn what flavor of consensus we want, what keys it must be + * signed with, and what diffs we would accept (or demand) instead. Return 0 + * on success and -1 on failure. + */ +static int +parse_consensus_request(parsed_consensus_request_t *out, + const get_handler_args_t *args) +{ + const char *url = args->url; + memset(out, 0, sizeof(parsed_consensus_request_t)); + out->flav = FLAV_NS; + + const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/"; + const char CONSENSUS_FLAVORED_PREFIX[] = + "/tor/status-vote/current/consensus-"; + + /* figure out the flavor if any, and who we wanted to sign the thing */ + const char *after_flavor = NULL; + + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + after_flavor = cp+1; + out->flavor = tor_strndup(f, cp-f); + } else { + out->flavor = tor_strdup(f); } + int flav = networkstatus_parse_flavor_name(out->flavor); + if (flav < 0) + flav = FLAV_NS; + out->flav = flav; + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + after_flavor = url+strlen(CONSENSUS_URL_PREFIX); + } - size_t dlen = dirserv_estimate_data_size(dir_fps, 0, compressed); - if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { - log_debug(LD_DIRSERV, - "Client asked for network status lists, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); - smartlist_free(dir_fps); + /* see whether we've been asked explicitly for a diff from an older + * consensus. (The user might also have said that a diff would be okay, + * via X-Or-Diff-From-Consensus */ + const char DIFF_COMPONENT[] = "diff/"; + char *diff_hash_in_url = NULL; + if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) { + after_flavor += strlen(DIFF_COMPONENT); + const char *cp = strchr(after_flavor, '/'); + if (cp) { + diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor); + out->want_fps = cp+1; + } else { + diff_hash_in_url = tor_strdup(after_flavor); + out->want_fps = NULL; + } + } else { + out->want_fps = after_flavor; + } - geoip_note_ns_response(GEOIP_REJECT_BUSY); - goto done; + if (diff_hash_in_url) { + uint8_t diff_from[DIGEST256_LEN]; + out->diff_from_digests = smartlist_new(); + out->diff_only = 1; + int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL", + "rejecting"); + tor_free(diff_hash_in_url); + if (ok) { + smartlist_add(out->diff_from_digests, + tor_memdup(diff_from, DIGEST256_LEN)); + } else { + return -1; } + } else { + parse_or_diff_from_header(&out->diff_from_digests, args->headers); + } - if (1) { - tor_addr_t addr; - if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, - &addr, NULL, - time(NULL)); - geoip_note_ns_response(GEOIP_SUCCESS); - /* Note that a request for a network status has started, so that we - * can measure the download time later on. */ - if (conn->dirreq_id) - geoip_start_dirreq(conn->dirreq_id, dlen, DIRREQ_TUNNELED); - else - geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, - DIRREQ_DIRECT); - } + return 0; +} + +/** Helper function for GET /tor/status-vote/current/consensus + */ +static int +handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 0); + const time_t if_modified_since = args->if_modified_since; + int clear_spool = 0; + + /* v3 network status fetch. */ + long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + + time_t now = time(NULL); + parsed_consensus_request_t req; + + if (parse_consensus_request(&req, args) < 0) { + write_short_http_response(conn, 404, "Couldn't parse request"); + goto done; + } + + if (digest_list_contains_best_consensus(req.flav, + req.diff_from_digests)) { + write_short_http_response(conn, 304, "Not modified"); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + goto done; + } + + struct consensus_cache_entry_t *cached_consensus = NULL; + + compress_method_t compression_used = NO_METHOD; + if (req.diff_from_digests) { + cached_consensus = find_best_diff(req.diff_from_digests, req.flav, + args->compression_supported, + &compression_used); + } + + if (req.diff_only && !cached_consensus) { + write_short_http_response(conn, 404, "No such diff available"); + // XXXX warn_consensus_is_too_old(v, req.flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } + + if (! cached_consensus) { + cached_consensus = find_best_consensus(req.flav, + args->compression_supported, + &compression_used); + } + + time_t fresh_until, valid_until; + int have_fresh_until = 0, have_valid_until = 0; + if (cached_consensus) { + have_fresh_until = + !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); + have_valid_until = + !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); + } + + if (cached_consensus && have_valid_until && + !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { + write_short_http_response(conn, 404, "Consensus is too old"); + warn_consensus_is_too_old(cached_consensus, req.flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } + + if (cached_consensus && req.want_fps && + !client_likes_consensus(cached_consensus, req.want_fps)) { + write_short_http_response(conn, 404, "Consensus not signed by sufficient " + "number of requested authorities"); + geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); + goto done; + } + + conn->spool = smartlist_new(); + clear_spool = 1; + { + spooled_resource_t *spooled; + if (cached_consensus) { + spooled = spooled_resource_new_from_cache_entry(cached_consensus); + smartlist_add(conn->spool, spooled); } + } - write_http_response_header(conn, -1, compressed, - smartlist_len(dir_fps) == 1 ? lifetime : 0); - conn->fingerprint_stack = dir_fps; - if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; - /* Prime the connection with some data. */ - conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS; - connection_dirserv_flushed_some(conn); + size_t size_guess = 0; + int n_expired = 0; + dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, + compress_method != NO_METHOD, + &size_guess, + &n_expired); + + if (!smartlist_len(conn->spool) && !n_expired) { + write_short_http_response(conn, 404, "Not found"); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } else if (!smartlist_len(conn->spool)) { + write_short_http_response(conn, 304, "Not modified"); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + goto done; + } + + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { + log_debug(LD_DIRSERV, + "Client asked for network status lists, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_short_http_response(conn, 503, "Directory busy, try again later"); + geoip_note_ns_response(GEOIP_REJECT_BUSY); goto done; } + tor_addr_t addr; + if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, + &addr, NULL, + time(NULL)); + geoip_note_ns_response(GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (conn->dirreq_id) + geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess, + DIRREQ_DIRECT); + } + + /* Use this header to tell caches that the response depends on the + * X-Or-Diff-From-Consensus header (or lack thereof). */ + const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n"; + + clear_spool = 0; + + // The compress_method might have been NO_METHOD, but we store the data + // compressed. Decompress them using `compression_used`. See fallback code in + // find_best_consensus() and find_best_diff(). + write_http_response_headers(conn, -1, + compress_method == NO_METHOD ? + NO_METHOD : compression_used, + vary_header, + smartlist_len(conn->spool) == 1 ? lifetime : 0); + + if (compress_method == NO_METHOD && smartlist_len(conn->spool)) + conn->compress_state = tor_compress_new(0, compression_used, + HIGH_COMPRESSION); + + /* Prime the connection with some data. */ + const int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); + goto done; + done: + parsed_consensus_request_clear(&req); + if (clear_spool) { + dir_conn_clear_spool(conn); + } return 0; } @@ -3026,12 +4438,14 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; { int current; ssize_t body_len = 0; ssize_t estimated_len = 0; + /* This smartlist holds strings that we can compress on the fly. */ smartlist_t *items = smartlist_new(); + /* This smartlist holds cached_dir_t objects that have a precompressed + * deflated version. */ smartlist_t *dir_items = smartlist_new(); int lifetime = 60; /* XXXX?? should actually use vote intervals. */ url += strlen("/tor/status-vote/"); @@ -3079,15 +4493,36 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) smartlist_free(fps); } if (!smartlist_len(dir_items) && !smartlist_len(items)) { - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); goto vote_done; } + + /* We're sending items from at most one kind of source */ + tor_assert_nonfatal(smartlist_len(items) == 0 || + smartlist_len(dir_items) == 0); + + int streaming; + unsigned mask; + if (smartlist_len(items)) { + /* We're taking strings and compressing them on the fly. */ + streaming = 1; + mask = ~0u; + } else { + /* We're taking cached_dir_t objects. We only have them uncompressed + * or deflated. */ + streaming = 0; + mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD); + } + const compress_method_t compress_method = find_best_compression_method( + args->compression_supported&mask, streaming); + SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compressed ? d->dir_z_len : d->dir_len); + body_len += compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); - if (compressed) { + if (compress_method != NO_METHOD) { estimated_len += ln/2; } else { body_len += ln; estimated_len += ln; @@ -3095,27 +4530,30 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) }); if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) { - write_http_status_line(conn, 503, "Directory busy, try again later"); + write_short_http_response(conn, 503, "Directory busy, try again later"); goto vote_done; } - write_http_response_header(conn, body_len ? body_len : -1, compressed, + write_http_response_header(conn, body_len ? body_len : -1, + compress_method, lifetime); if (smartlist_len(items)) { - if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(estimated_len)); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf_zlib(c, strlen(c), conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_buf_add_compress(c, strlen(c), conn, 0)); + connection_buf_add_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf(c, strlen(c), TO_CONN(conn))); + connection_buf_add(c, strlen(c), TO_CONN(conn))); } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compressed ? d->dir_z : d->dir, - compressed ? d->dir_z_len : d->dir_len, + connection_buf_add(compress_method != NO_METHOD ? + d->dir_compressed : d->dir, + compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } vote_done: @@ -3133,44 +4571,51 @@ static int handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + int clear_spool = 1; { - smartlist_t *fps = smartlist_new(); + conn->spool = smartlist_new(); - dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), - fps, NULL, + dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"), + DIR_SPOOL_MICRODESC, + conn->spool, NULL, DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); - if (!dirserv_have_any_microdesc(fps)) { - write_http_status_line(conn, 404, "Not found"); - SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); - smartlist_free(fps); + size_t size_guess = 0; + dirserv_spool_remove_missing_and_guess_size(conn, 0, + compress_method != NO_METHOD, + &size_guess, NULL); + if (smartlist_len(conn->spool) == 0) { + write_short_http_response(conn, 404, "Not found"); goto done; } - size_t dlen = dirserv_estimate_microdesc_size(fps, compressed); - if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { log_info(LD_DIRSERV, "Client asked for server descriptors, but we've been " "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); - smartlist_free(fps); + write_short_http_response(conn, 503, "Directory busy, try again later"); goto done; } - write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); - conn->dir_spool_src = DIR_SPOOL_MICRODESC; - conn->fingerprint_stack = fps; + clear_spool = 0; + write_http_response_header(conn, -1, + compress_method, + MICRODESC_CACHE_LIFETIME); - if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(dlen)); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(size_guess)); - connection_dirserv_flushed_some(conn); + const int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); goto done; } done: + if (clear_spool) { + dir_conn_clear_spool(conn); + } return 0; } @@ -3180,71 +4625,93 @@ static int handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); const or_options_t *options = get_options(); + int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { - size_t dlen; int res; - const char *msg; + const char *msg = NULL; int cache_lifetime = 0; int is_extra = !strcmpstart(url,"/tor/extra/"); url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/"); - conn->fingerprint_stack = smartlist_new(); - res = dirserv_get_routerdesc_fingerprints(conn->fingerprint_stack, url, - &msg, - !connection_dir_is_encrypted(conn), - is_extra); - - if (!strcmpstart(url, "fp/")) { - if (smartlist_len(conn->fingerprint_stack) == 1) - cache_lifetime = ROUTERDESC_CACHE_LIFETIME; - } else if (!strcmpstart(url, "authority")) { - cache_lifetime = ROUTERDESC_CACHE_LIFETIME; - } else if (!strcmpstart(url, "all")) { - cache_lifetime = FULL_DIR_CACHE_LIFETIME; - } else if (!strcmpstart(url, "d/")) { - if (smartlist_len(conn->fingerprint_stack) == 1) - cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; - } - if (!strcmpstart(url, "d/")) - conn->dir_spool_src = + dir_spool_source_t source; + time_t publish_cutoff = 0; + if (!strcmpstart(url, "d/")) { + source = is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST; - else - conn->dir_spool_src = + } else { + source = is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP; + /* We only want to apply a publish cutoff when we're requesting + * resources by fingerprint. */ + publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; + } + + conn->spool = smartlist_new(); + res = dirserv_get_routerdesc_spool(conn->spool, url, + source, + connection_dir_is_encrypted(conn), + &msg); + + if (!strcmpstart(url, "all")) { + cache_lifetime = FULL_DIR_CACHE_LIFETIME; + } else if (smartlist_len(conn->spool) == 1) { + cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; + } - if (!dirserv_have_any_serverdesc(conn->fingerprint_stack, - conn->dir_spool_src)) { - res = -1; - msg = "Not found"; + size_t size_guess = 0; + int n_expired = 0; + dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff, + compress_method != NO_METHOD, + &size_guess, &n_expired); + + /* If we are the bridge authority and the descriptor is a bridge + * descriptor, remember that we served this descriptor for desc stats. */ + /* XXXX it's a bit of a kludge to have this here. */ + if (get_options()->BridgeAuthoritativeDir && + source == DIR_SPOOL_SERVER_BY_FP) { + SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) { + const routerinfo_t *router = + router_get_by_id_digest((const char *)spooled->digest); + /* router can be NULL here when the bridge auth is asked for its own + * descriptor. */ + if (router && router->purpose == ROUTER_PURPOSE_BRIDGE) + rep_hist_note_desc_served(router->cache_info.identity_digest); + } SMARTLIST_FOREACH_END(spooled); } - if (res < 0) - write_http_status_line(conn, 404, msg); - else { - dlen = dirserv_estimate_data_size(conn->fingerprint_stack, - 1, compressed); - if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) { + if (msg == NULL) + msg = "Not found"; + write_short_http_response(conn, 404, msg); + } else { + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { log_info(LD_DIRSERV, "Client asked for server descriptors, but we've been " "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - conn->dir_spool_src = DIR_SPOOL_NONE; + write_short_http_response(conn, 503, + "Directory busy, try again later"); + dir_conn_clear_spool(conn); goto done; } - write_http_response_header(conn, -1, compressed, cache_lifetime); - if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(dlen)); + write_http_response_header(conn, -1, compress_method, cache_lifetime); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(size_guess)); + clear_spool = 0; /* Prime the connection with some data. */ - connection_dirserv_flushed_some(conn); + int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); } goto done; } done: - return 0; + if (clear_spool) + dir_conn_clear_spool(conn); + return 0; } /** Helper function for GET /tor/keys/... @@ -3253,7 +4720,8 @@ static int handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compressed; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new(); @@ -3298,41 +4766,47 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) }); smartlist_free(fp_sks); } else { - write_http_status_line(conn, 400, "Bad request"); + write_short_http_response(conn, 400, "Bad request"); goto keys_done; } if (!smartlist_len(certs)) { - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); goto keys_done; } SMARTLIST_FOREACH(certs, authority_cert_t *, c, if (c->cache_info.published_on < if_modified_since) SMARTLIST_DEL_CURRENT(certs, c)); if (!smartlist_len(certs)) { - write_http_status_line(conn, 304, "Not modified"); + write_short_http_response(conn, 304, "Not modified"); goto keys_done; } len = 0; SMARTLIST_FOREACH(certs, authority_cert_t *, c, len += c->cache_info.signed_descriptor_len); - if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) { - write_http_status_line(conn, 503, "Directory busy, try again later"); + if (global_write_bucket_low(TO_CONN(conn), + compress_method != NO_METHOD ? len/2 : len, + 2)) { + write_short_http_response(conn, 503, "Directory busy, try again later"); goto keys_done; } - write_http_response_header(conn, compressed?-1:len, compressed, 60*60); - if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(len)); + write_http_response_header(conn, + compress_method != NO_METHOD ? -1 : len, + compress_method, + 60*60); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_buf_add_compress( + c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, 0)); + connection_buf_add_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf(c->cache_info.signed_descriptor_body, + connection_buf_add(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, TO_CONN(conn))); } @@ -3347,7 +4821,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) /** Helper function for GET /tor/rendezvous2/ */ static int -handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args) +handle_get_hs_descriptor_v2(dir_connection_t *conn, + const get_handler_args_t *args) { const char *url = args->url; if (connection_dir_is_encrypted(conn)) { @@ -3359,28 +4834,65 @@ handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args) safe_str(escaped(query))); switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { case 1: /* valid */ - write_http_response_header(conn, strlen(descp), 0, 0); - connection_write_to_buf(descp, strlen(descp), TO_CONN(conn)); + write_http_response_header(conn, strlen(descp), NO_METHOD, 0); + connection_buf_add(descp, strlen(descp), TO_CONN(conn)); break; case 0: /* well-formed but not present */ - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); break; case -1: /* not well-formed */ - write_http_status_line(conn, 400, "Bad request"); + write_short_http_response(conn, 400, "Bad request"); break; } } else { /* not well-formed */ - write_http_status_line(conn, 400, "Bad request"); + write_short_http_response(conn, 400, "Bad request"); } goto done; } else { /* Not encrypted! */ - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); } done: return 0; } +/** Helper function for GET /tor/hs/3/<z>. Only for version 3. + */ +STATIC int +handle_get_hs_descriptor_v3(dir_connection_t *conn, + const get_handler_args_t *args) +{ + int retval; + const char *desc_str = NULL; + const char *pubkey_str = NULL; + const char *url = args->url; + + /* Reject unencrypted dir connections */ + if (!connection_dir_is_encrypted(conn)) { + write_short_http_response(conn, 404, "Not found"); + goto done; + } + + /* After the path prefix follows the base64 encoded blinded pubkey which we + * use to get the descriptor from the cache. Skip the prefix and get the + * pubkey. */ + tor_assert(!strcmpstart(url, "/tor/hs/3/")); + pubkey_str = url + strlen("/tor/hs/3/"); + retval = hs_cache_lookup_as_dir(HS_VERSION_THREE, + pubkey_str, &desc_str); + if (retval <= 0 || desc_str == NULL) { + write_short_http_response(conn, 404, "Not found"); + goto done; + } + + /* Found requested descriptor! Pass it to this nice client. */ + write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0); + connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn)); + + done: + return 0; +} + /** Helper function for GET /tor/networkstatus-bridges */ static int @@ -3404,7 +4916,7 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, if (!header || tor_memneq(digest, options->BridgePassword_AuthDigest_, DIGEST256_LEN)) { - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); tor_free(header); goto done; } @@ -3413,8 +4925,8 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, /* all happy now. send an answer. */ status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); size_t dlen = strlen(status); - write_http_response_header(conn, dlen, 0, 0); - connection_write_to_buf(status, dlen, TO_CONN(conn)); + write_http_response_header(conn, dlen, NO_METHOD, 0); + connection_buf_add(status, dlen, TO_CONN(conn)); tor_free(status); goto done; } @@ -3430,20 +4942,104 @@ handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) { const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; size_t len = strlen(robots); - write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME); - connection_write_to_buf(robots, len, TO_CONN(conn)); + write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME); + connection_buf_add(robots, len, TO_CONN(conn)); } return 0; } +/* Given the <b>url</b> from a POST request, try to extract the version number + * using the provided <b>prefix</b>. The version should be after the prefix and + * ending with the separator "/". For instance: + * /tor/hs/3/publish + * + * On success, <b>end_pos</b> points to the position right after the version + * was found. On error, it is set to NULL. + * + * Return version on success else negative value. */ +STATIC int +parse_hs_version_from_post(const char *url, const char *prefix, + const char **end_pos) +{ + int ok; + unsigned long version; + const char *start; + char *end = NULL; + + tor_assert(url); + tor_assert(prefix); + tor_assert(end_pos); + + /* Check if the prefix does start the url. */ + if (strcmpstart(url, prefix)) { + goto err; + } + /* Move pointer to the end of the prefix string. */ + start = url + strlen(prefix); + /* Try this to be the HS version and if we are still at the separator, next + * will be move to the right value. */ + version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end); + if (!ok) { + goto err; + } + + *end_pos = end; + return (int) version; + err: + *end_pos = NULL; + return -1; +} + +/* Handle the POST request for a hidden service descripror. The request is in + * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success + * else return 400 indicating a bad request. */ +STATIC int +handle_post_hs_descriptor(const char *url, const char *body) +{ + int version; + const char *end_pos; + + tor_assert(url); + tor_assert(body); + + version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos); + if (version < 0) { + goto err; + } + + /* We have a valid version number, now make sure it's a publish request. Use + * the end position just after the version and check for the command. */ + if (strcmpstart(end_pos, "/publish")) { + goto err; + } + + switch (version) { + case HS_VERSION_THREE: + if (hs_cache_store_as_dir(body) < 0) { + goto err; + } + log_info(LD_REND, "Publish request for HS descriptor handled " + "successfully."); + break; + default: + /* Unsupported version, return a bad request. */ + goto err; + } + + return 200; + err: + /* Bad request. */ + return 400; +} + /** Helper function: called when a dirserver gets a complete HTTP POST * request. Look for an uploaded server descriptor or rendezvous * service descriptor. On finding one, process it and write a * response into conn-\>outbuf. If the request is unrecognized, send a * 400. Always return 0. */ -static int -directory_handle_command_post(dir_connection_t *conn, const char *headers, - const char *body, size_t body_len) +MOCK_IMPL(STATIC int, +directory_handle_command_post,(dir_connection_t *conn, const char *headers, + const char *body, size_t body_len)) { char *url = NULL; const or_options_t *options = get_options(); @@ -3455,12 +5051,12 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, if (!public_server_mode(options)) { log_info(LD_DIR, "Rejected dir post request from %s " "since we're not a public relay.", conn->base_.address); - write_http_status_line(conn, 503, "Not acting as a public relay"); + write_short_http_response(conn, 503, "Not acting as a public relay"); goto done; } if (parse_http_url(headers, &url) < 0) { - write_http_status_line(conn, 400, "Bad request"); + write_short_http_response(conn, 400, "Bad request"); return 0; } log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); @@ -3469,26 +5065,41 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, if (connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/publish")) { if (rend_cache_store_v2_desc_as_dir(body) < 0) { - log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", + log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.", (int)body_len, conn->base_.address); - write_http_status_line(conn, 400, + write_short_http_response(conn, 400, "Invalid v2 service descriptor rejected"); } else { - write_http_status_line(conn, 200, "Service descriptor (v2) stored"); + write_short_http_response(conn, 200, "Service descriptor (v2) stored"); log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); } goto done; } + /* Handle HS descriptor publish request. */ + /* XXX: This should be disabled with a consensus param until we want to + * the prop224 be deployed and thus use. */ + if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) { + const char *msg = "HS descriptor stored successfully."; + + /* We most probably have a publish request for an HS descriptor. */ + int code = handle_post_hs_descriptor(url, body); + if (code != 200) { + msg = "Invalid HS descriptor. Rejected."; + } + write_short_http_response(conn, code, msg); + goto done; + } + if (!authdir_mode(options)) { /* we just provide cached directories; we don't want to * receive anything. */ - write_http_status_line(conn, 400, "Nonauthoritative directory does not " + write_short_http_response(conn, 400, "Nonauthoritative directory does not " "accept posted server descriptors"); goto done; } - if (authdir_mode_handles_descs(options, -1) && + if (authdir_mode(options) && !strcmp(url,"/tor/")) { /* server descriptor post */ const char *msg = "[None]"; uint8_t purpose = authdir_mode_bridge(options) ? @@ -3497,15 +5108,8 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, conn->base_.address, &msg); tor_assert(msg); - if (r == ROUTER_ADDED_NOTIFY_GENERATOR) { - /* Accepted with a message. */ - log_info(LD_DIRSERV, - "Problematic router descriptor or extra-info from %s " - "(\"%s\").", - conn->base_.address, msg); - write_http_status_line(conn, 400, msg); - } else if (r == ROUTER_ADDED_SUCCESSFULLY) { - write_http_status_line(conn, 200, msg); + if (r == ROUTER_ADDED_SUCCESSFULLY) { + write_short_http_response(conn, 200, msg); } else if (WRA_WAS_OUTDATED(r)) { write_http_response_header_impl(conn, -1, NULL, NULL, "X-Descriptor-Not-New: Yes\r\n", -1); @@ -3514,7 +5118,7 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, "Rejected router descriptor or extra-info from %s " "(\"%s\").", conn->base_.address, msg); - write_http_status_line(conn, 400, msg); + write_short_http_response(conn, 400, msg); } goto done; } @@ -3524,12 +5128,12 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, const char *msg = "OK"; int status; if (dirvote_add_vote(body, &msg, &status)) { - write_http_status_line(conn, status, "Vote stored"); + write_short_http_response(conn, status, "Vote stored"); } else { tor_assert(msg); log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").", conn->base_.address, msg); - write_http_status_line(conn, status, msg); + write_short_http_response(conn, status, msg); } goto done; } @@ -3538,17 +5142,18 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */ const char *msg = NULL; if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) { - write_http_status_line(conn, 200, msg?msg:"Signatures stored"); + write_short_http_response(conn, 200, msg?msg:"Signatures stored"); } else { log_warn(LD_DIR, "Unable to store signatures posted by %s: %s", conn->base_.address, msg?msg:"???"); - write_http_status_line(conn, 400, msg?msg:"Unable to store signatures"); + write_short_http_response(conn, 400, + msg?msg:"Unable to store signatures"); } goto done; } /* we didn't recognize the url */ - write_http_status_line(conn, 404, "Not found"); + write_short_http_response(conn, 404, "Not found"); done: tor_free(url); @@ -3560,7 +5165,7 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, * from the inbuf, try to process it; otherwise, leave it on the * buffer. Return a 0 on success, or -1 on error. */ -static int +STATIC int directory_handle_command(dir_connection_t *conn) { char *headers=NULL, *body=NULL; @@ -3631,7 +5236,7 @@ connection_dir_finished_flushing(dir_connection_t *conn) conn->base_.state = DIR_CONN_STATE_CLIENT_READING; return 0; case DIR_CONN_STATE_SERVER_WRITING: - if (conn->dir_spool_src != DIR_SPOOL_NONE) { + if (conn->spool) { log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!"); connection_mark_for_close(TO_CONN(conn)); } else { @@ -3701,7 +5306,7 @@ connection_dir_finished_connecting(dir_connection_t *conn) * Helper function for download_status_increment_failure(), * download_status_reset(), and download_status_increment_attempt(). */ STATIC const smartlist_t * -find_dl_schedule(download_status_t *dls, const or_options_t *options) +find_dl_schedule(const download_status_t *dls, const or_options_t *options) { switch (dls->schedule) { case DL_SCHED_GENERIC: @@ -3742,7 +5347,15 @@ find_dl_schedule(download_status_t *dls, const or_options_t *options) } } case DL_SCHED_BRIDGE: - return options->TestingBridgeDownloadSchedule; + if (options->UseBridges && num_bridges_usable(0) > 0) { + /* A bridge client that is sure that one or more of its bridges are + * running can afford to wait longer to update bridge descriptors. */ + return options->TestingBridgeDownloadSchedule; + } else { + /* A bridge client which might have no running bridges, must try to + * get bridge descriptors straight away. */ + return options->TestingBridgeBootstrapDownloadSchedule; + } default: tor_assert(0); } @@ -3751,17 +5364,14 @@ find_dl_schedule(download_status_t *dls, const or_options_t *options) return NULL; } -/** Decide which minimum and maximum delay step we want to use based on +/** Decide which minimum delay step we want to use based on * descriptor type in <b>dls</b> and <b>options</b>. * Helper function for download_status_schedule_get_delay(). */ -STATIC void -find_dl_min_and_max_delay(download_status_t *dls, const or_options_t *options, - int *min, int *max) +STATIC int +find_dl_min_delay(download_status_t *dls, const or_options_t *options) { tor_assert(dls); tor_assert(options); - tor_assert(min); - tor_assert(max); /* * For now, just use the existing schedule config stuff and pick the @@ -3769,82 +5379,77 @@ find_dl_min_and_max_delay(download_status_t *dls, const or_options_t *options, */ const smartlist_t *schedule = find_dl_schedule(dls, options); tor_assert(schedule != NULL && smartlist_len(schedule) >= 2); - *min = *((int *)(smartlist_get(schedule, 0))); - if (dls->backoff == DL_SCHED_DETERMINISTIC) - *max = *((int *)((smartlist_get(schedule, smartlist_len(schedule) - 1)))); - else - *max = INT_MAX; + return *(int *)(smartlist_get(schedule, 0)); +} + +/** As next_random_exponential_delay() below, but does not compute a random + * value. Instead, compute the range of values that + * next_random_exponential_delay() should use when computing its random value. + * Store the low bound into *<b>low_bound_out</b>, and the high bound into + * *<b>high_bound_out</b>. Guarantees that the low bound is strictly less + * than the high bound. */ +STATIC void +next_random_exponential_delay_range(int *low_bound_out, + int *high_bound_out, + int delay, + int base_delay) +{ + // This is the "decorrelated jitter" approach, from + // https://www.awsarchitectureblog.com/2015/03/backoff.html + // The formula is + // sleep = min(cap, random_between(base, sleep * 3)) + + const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX; + *low_bound_out = base_delay; + if (delay_times_3 > base_delay) { + *high_bound_out = delay_times_3; + } else { + *high_bound_out = base_delay+1; + } } -/** Advance one delay step. The algorithm is to use the previous delay to - * compute an increment, we construct a value uniformly at random between - * delay and MAX(delay*2,delay+1). We then clamp that value to be no larger - * than max_delay, and return it. +/** Advance one delay step. The algorithm will generate a random delay, + * such that each failure is possibly (random) longer than the ones before. + * + * We then clamp that value to be no larger than max_delay, and return it. * - * Requires that delay is less than INT_MAX, and delay is in [0,max_delay]. + * The <b>base_delay</b> parameter is lowest possible delay time (can't be + * zero); the <b>backoff_position</b> parameter is the number of times we've + * generated a delay; and the <b>delay</b> argument is the most recently used + * delay. */ STATIC int -next_random_exponential_delay(int delay, int max_delay) +next_random_exponential_delay(int delay, + int base_delay) { /* Check preconditions */ - if (BUG(max_delay < 0)) - max_delay = 0; - if (BUG(delay > max_delay)) - delay = max_delay; - if (delay == INT_MAX) - return INT_MAX; /* prevent overflow */ if (BUG(delay < 0)) delay = 0; - /* How much are we willing to add to the delay? */ - int max_increment; - int multiplier = 3; /* no more than quadruple the previous delay */ - if (get_options()->TestingTorNetwork) { - /* Decrease the multiplier in testing networks. This reduces the variance, - * so that bootstrap is more reliable. */ - multiplier = 2; /* no more than triple the previous delay */ - } - - if (delay && delay < (INT_MAX-1) / multiplier) { - max_increment = delay * multiplier; - } else if (delay) { - max_increment = INT_MAX-1; - } else { - max_increment = 1; - } + if (base_delay < 1) + base_delay = 1; - if (BUG(max_increment < 1)) - max_increment = 1; + int low_bound=0, high_bound=INT_MAX; - /* the + 1 here is so that we always wait longer than last time. */ - int increment = crypto_rand_int(max_increment)+1; + next_random_exponential_delay_range(&low_bound, &high_bound, + delay, base_delay); - if (increment < max_delay - delay) - return delay + increment; - else - return max_delay; + return crypto_rand_int_range(low_bound, high_bound); } -/** Find the current delay for dls based on schedule or min_delay/ - * max_delay if we're using exponential backoff. If dls->backoff is - * DL_SCHED_RANDOM_EXPONENTIAL, we must have 0 <= min_delay <= max_delay <= - * INT_MAX, but schedule may be set to NULL; otherwise schedule is required. +/** Find the current delay for dls based on min_delay. + * * This function sets dls->next_attempt_at based on now, and returns the delay. * Helper for download_status_increment_failure and * download_status_increment_attempt. */ STATIC int download_status_schedule_get_delay(download_status_t *dls, - const smartlist_t *schedule, - int min_delay, int max_delay, + int min_delay, time_t now) { tor_assert(dls); - /* We don't need a schedule if we're using random exponential backoff */ - tor_assert(dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL || - schedule != NULL); /* If we're using random exponential backoff, we do need min/max delay */ - tor_assert(dls->backoff != DL_SCHED_RANDOM_EXPONENTIAL || - (min_delay >= 0 && max_delay >= min_delay)); + tor_assert(min_delay >= 0); int delay = INT_MAX; uint8_t dls_schedule_position = (dls->increment_on @@ -3852,42 +5457,32 @@ download_status_schedule_get_delay(download_status_t *dls, ? dls->n_download_attempts : dls->n_download_failures); - if (dls->backoff == DL_SCHED_DETERMINISTIC) { - if (dls_schedule_position < smartlist_len(schedule)) - delay = *(int *)smartlist_get(schedule, dls_schedule_position); - else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD) - delay = INT_MAX; - else - delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); - } else if (dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL) { - /* Check if we missed a reset somehow */ - if (dls->last_backoff_position > dls_schedule_position) { - dls->last_backoff_position = 0; - dls->last_delay_used = 0; - } + /* Check if we missed a reset somehow */ + IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) { + dls->last_backoff_position = 0; + dls->last_delay_used = 0; + } - if (dls_schedule_position > 0) { - delay = dls->last_delay_used; + if (dls_schedule_position > 0) { + delay = dls->last_delay_used; - while (dls->last_backoff_position < dls_schedule_position) { - /* Do one increment step */ - delay = next_random_exponential_delay(delay, max_delay); - /* Update our position */ - ++(dls->last_backoff_position); - } - } else { - /* If we're just starting out, use the minimum delay */ - delay = min_delay; + while (dls->last_backoff_position < dls_schedule_position) { + /* Do one increment step */ + delay = next_random_exponential_delay(delay, min_delay); + /* Update our position */ + ++(dls->last_backoff_position); } + } else { + /* If we're just starting out, use the minimum delay */ + delay = min_delay; + } - /* Clamp it within min/max if we have them */ - if (min_delay >= 0 && delay < min_delay) delay = min_delay; - if (max_delay != INT_MAX && delay > max_delay) delay = max_delay; + /* Clamp it within min/max if we have them */ + if (min_delay >= 0 && delay < min_delay) delay = min_delay; - /* Store it for next time */ - dls->last_backoff_position = dls_schedule_position; - dls->last_delay_used = delay; - } + /* Store it for next time */ + dls->last_backoff_position = dls_schedule_position; + dls->last_delay_used = delay; /* A negative delay makes no sense. Knowing that delay is * non-negative allows us to safely do the wrapping check below. */ @@ -3950,10 +5545,15 @@ download_status_increment_failure(download_status_t *dls, int status_code, (void) status_code; // XXXX no longer used. (void) server; // XXXX no longer used. int increment = -1; - int min_delay = 0, max_delay = INT_MAX; + int min_delay = 0; tor_assert(dls); + /* dls wasn't reset before it was used */ + if (dls->next_attempt_at == 0) { + download_status_reset(dls); + } + /* count the failure */ if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) { ++dls->n_download_failures; @@ -3970,22 +5570,22 @@ download_status_increment_failure(download_status_t *dls, int status_code, /* only return a failure retry time if this schedule increments on failures */ - const smartlist_t *schedule = find_dl_schedule(dls, get_options()); - find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay); - increment = download_status_schedule_get_delay(dls, schedule, - min_delay, max_delay, now); + min_delay = find_dl_min_delay(dls, get_options()); + increment = download_status_schedule_get_delay(dls, min_delay, now); } download_status_log_helper(item, !dls->increment_on, "failed", "concurrently", dls->n_download_failures, - increment, dls->next_attempt_at, now); + increment, + download_status_get_next_attempt_at(dls), + now); if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) { /* stop this schedule retrying on failure, it will launch concurrent * connections instead */ return TIME_MAX; } else { - return dls->next_attempt_at; + return download_status_get_next_attempt_at(dls); } } @@ -4002,10 +5602,15 @@ download_status_increment_attempt(download_status_t *dls, const char *item, time_t now) { int delay = -1; - int min_delay = 0, max_delay = INT_MAX; + int min_delay = 0; tor_assert(dls); + /* dls wasn't reset before it was used */ + if (dls->next_attempt_at == 0) { + download_status_reset(dls); + } + if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { /* this schedule should retry on failure, and not launch any concurrent attempts */ @@ -4017,16 +5622,24 @@ download_status_increment_attempt(download_status_t *dls, const char *item, if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) ++dls->n_download_attempts; - const smartlist_t *schedule = find_dl_schedule(dls, get_options()); - find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay); - delay = download_status_schedule_get_delay(dls, schedule, - min_delay, max_delay, now); + min_delay = find_dl_min_delay(dls, get_options()); + delay = download_status_schedule_get_delay(dls, min_delay, now); download_status_log_helper(item, dls->increment_on, "attempted", "on failure", dls->n_download_attempts, - delay, dls->next_attempt_at, now); + delay, download_status_get_next_attempt_at(dls), + now); - return dls->next_attempt_at; + return download_status_get_next_attempt_at(dls); +} + +static time_t +download_status_get_initial_delay_from_now(const download_status_t *dls) +{ + const smartlist_t *schedule = find_dl_schedule(dls, get_options()); + /* We use constant initial delays, even in exponential backoff + * schedules. */ + return time(NULL) + *(int *)smartlist_get(schedule, 0); } /** Reset <b>dls</b> so that it will be considered downloadable @@ -4038,8 +5651,8 @@ download_status_increment_attempt(download_status_t *dls, const char *item, * (We find the zeroth element of the download schedule, and set * next_attempt_at to be the appropriate offset from 'now'. In most * cases this means setting it to 'now', so the item will be immediately - * downloadable; in the case of bridge descriptors, the zeroth element - * is an hour from now.) */ + * downloadable; when using authorities with fallbacks, there is a few seconds' + * delay.) */ void download_status_reset(download_status_t *dls) { @@ -4047,11 +5660,9 @@ download_status_reset(download_status_t *dls) || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD) return; /* Don't reset this. */ - const smartlist_t *schedule = find_dl_schedule(dls, get_options()); - dls->n_download_failures = 0; dls->n_download_attempts = 0; - dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0); + dls->next_attempt_at = download_status_get_initial_delay_from_now(dls); dls->last_backoff_position = 0; dls->last_delay_used = 0; /* Don't reset dls->want_authority or dls->increment_on */ @@ -4078,6 +5689,12 @@ download_status_get_n_attempts(const download_status_t *dls) time_t download_status_get_next_attempt_at(const download_status_t *dls) { + /* dls wasn't reset before it was used */ + if (dls->next_attempt_at == 0) { + /* so give the answer we would have given if it had been */ + return download_status_get_initial_delay_from_now(dls); + } + return dls->next_attempt_at; } @@ -4115,8 +5732,7 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, } else { dls = router_get_dl_status_by_descriptor_digest(digest); } - if (!dls || dls->n_download_failures >= - get_options()->TestingDescriptorMaxDownloadTries) + if (!dls) continue; download_status_increment_failure(dls, status_code, cp, server, now); } SMARTLIST_FOREACH_END(cp); @@ -4125,13 +5741,14 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */ } -/** Called when a connection to download microdescriptors has failed in whole - * or in part. <b>failed</b> is a list of every microdesc digest we didn't - * get. <b>status_code</b> is the http status code we received. Reschedule the - * microdesc downloads as appropriate. */ +/** Called when a connection to download microdescriptors from relay with + * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list + * of every microdesc digest we didn't get. <b>status_code</b> is the http + * status code we received. Reschedule the microdesc downloads as + * appropriate. */ static void dir_microdesc_download_failed(smartlist_t *failed, - int status_code) + int status_code, const char *dir_id) { networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC); @@ -4142,17 +5759,22 @@ dir_microdesc_download_failed(smartlist_t *failed, if (! consensus) return; + + /* We failed to fetch a microdescriptor from 'dir_id', note it down + * so that we don't try the same relay next time... */ + microdesc_note_outdated_dirserver(dir_id); + SMARTLIST_FOREACH_BEGIN(failed, const char *, d) { rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d); if (!rs) continue; dls = &rs->dl_status; - if (dls->n_download_failures >= - get_options()->TestingMicrodescMaxDownloadTries) - continue; - { + + { /* Increment the failure count for this md fetch */ char buf[BASE64_DIGEST256_LEN+1]; digest256_to_base64(buf, d); + log_info(LD_DIR, "Failed to download md %s from %s", + buf, hex_str(dir_id, DIGEST_LEN)); download_status_increment_failure(dls, status_code, buf, server, now); } @@ -4314,3 +5936,34 @@ dir_split_resource_into_fingerprints(const char *resource, return 0; } +/** As dir_split_resource_into_fingerprints, but instead fills + * <b>spool_out</b> with a list of spoolable_resource_t for the resource + * identified through <b>source</b>. */ +int +dir_split_resource_into_spoolable(const char *resource, + dir_spool_source_t source, + smartlist_t *spool_out, + int *compressed_out, + int flags) +{ + smartlist_t *fingerprints = smartlist_new(); + + tor_assert(flags & (DSR_HEX|DSR_BASE64)); + const size_t digest_len = + (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN; + + int r = dir_split_resource_into_fingerprints(resource, fingerprints, + compressed_out, flags); + /* This is not a very efficient implementation XXXX */ + SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) { + spooled_resource_t *spooled = + spooled_resource_new(source, digest, digest_len); + if (spooled) + smartlist_add(spool_out, spooled); + tor_free(digest); + } SMARTLIST_FOREACH_END(digest); + + smartlist_free(fingerprints); + return r; +} + |