diff options
Diffstat (limited to 'src/feature/dirclient/dirclient.c')
| -rw-r--r-- | src/feature/dirclient/dirclient.c | 3206 |
1 files changed, 3206 insertions, 0 deletions
diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c new file mode 100644 index 0000000000..0fd1a47017 --- /dev/null +++ b/src/feature/dirclient/dirclient.c @@ -0,0 +1,3206 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define DIRCLIENT_PRIVATE + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" +#include "core/or/connection_edge.h" +#include "core/or/policies.h" +#include "feature/client/bridges.h" +#include "feature/client/entrynodes.h" +#include "feature/control/control.h" +#include "feature/dirauth/authmode.h" +#include "feature/dirauth/dirvote.h" +#include "feature/dirauth/shared_random.h" +#include "feature/dircache/dirserv.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" +#include "feature/dircommon/consdiff.h" +#include "feature/dircommon/directory.h" +#include "feature/dircommon/fp_pair.h" +#include "feature/hs/hs_cache.h" +#include "feature/hs/hs_client.h" +#include "feature/hs/hs_control.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/describe.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/microdesc.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerinfo.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerset.h" +#include "feature/relay/routermode.h" +#include "feature/relay/selftest.h" +#include "feature/rend/rendcache.h" +#include "feature/rend/rendclient.h" +#include "feature/rend/rendcommon.h" +#include "feature/rend/rendservice.h" +#include "feature/stats/predict_ports.h" + +#include "lib/compress/compress.h" +#include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/encoding/confline.h" +#include "lib/err/backtrace.h" + +#include "core/or/entry_connection_st.h" +#include "feature/dircache/cached_dir_st.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/dircommon/dir_connection_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/rend/rend_service_descriptor_st.h" + +/** Maximum size, in bytes, for any directory object that we've downloaded. */ +#define MAX_DIR_DL_SIZE ((1<<24)-1) /* 16 MB - 1 */ + +/** How far in the future do we allow a directory server to tell us it is + * before deciding that one of us has the wrong time? */ +#define ALLOW_DIRECTORY_TIME_SKEW (30*60) + +static int body_is_plausible(const char *body, size_t body_len, int purpose); +static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); +static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); +static void connection_dir_download_cert_failed( + dir_connection_t *conn, int status_code); +static void connection_dir_retry_bridges(smartlist_t *descs); +static void dir_routerdesc_download_failed(smartlist_t *failed, + int status_code, + int router_purpose, + int was_extrainfo, + int was_descriptor_digests); +static void dir_microdesc_download_failed(smartlist_t *failed, + int status_code, + const char *dir_id); +static void directory_send_command(dir_connection_t *conn, + const int direct, + const directory_request_t *req); +static void connection_dir_close_consensus_fetches( + dir_connection_t *except_this_one, const char *resource); + +/** Return a string describing a given directory connection purpose. */ +STATIC const char * +dir_conn_purpose_to_string(int purpose) +{ + switch (purpose) + { + case DIR_PURPOSE_UPLOAD_DIR: + return "server descriptor upload"; + case DIR_PURPOSE_UPLOAD_VOTE: + return "server vote upload"; + case DIR_PURPOSE_UPLOAD_SIGNATURES: + return "consensus signature upload"; + case DIR_PURPOSE_FETCH_SERVERDESC: + return "server descriptor fetch"; + case DIR_PURPOSE_FETCH_EXTRAINFO: + return "extra-info fetch"; + case DIR_PURPOSE_FETCH_CONSENSUS: + return "consensus network-status fetch"; + case DIR_PURPOSE_FETCH_CERTIFICATE: + return "authority cert fetch"; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + return "status vote fetch"; + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + return "consensus signature fetch"; + case DIR_PURPOSE_FETCH_RENDDESC_V2: + return "hidden-service v2 descriptor fetch"; + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + return "hidden-service v2 descriptor upload"; + case DIR_PURPOSE_FETCH_HSDESC: + return "hidden-service descriptor fetch"; + case DIR_PURPOSE_UPLOAD_HSDESC: + return "hidden-service descriptor upload"; + case DIR_PURPOSE_FETCH_MICRODESC: + return "microdescriptor fetch"; + } + + log_warn(LD_BUG, "Called with unknown purpose %d", purpose); + return "(unknown)"; +} + +/** Return the requisite directory information types. */ +STATIC dirinfo_type_t +dir_fetch_type(int dir_purpose, int router_purpose, const char *resource) +{ + dirinfo_type_t type; + switch (dir_purpose) { + case DIR_PURPOSE_FETCH_EXTRAINFO: + type = EXTRAINFO_DIRINFO; + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type |= BRIDGE_DIRINFO; + else + type |= V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type = BRIDGE_DIRINFO; + else + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_CERTIFICATE: + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_CONSENSUS: + type = V3_DIRINFO; + if (resource && !strcmp(resource, "microdesc")) + type |= MICRODESC_DIRINFO; + break; + case DIR_PURPOSE_FETCH_MICRODESC: + type = MICRODESC_DIRINFO; + break; + default: + log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); + type = NO_DIRINFO; + break; + } + return type; +} + +/** Return true iff <b>identity_digest</b> is the digest of a router which + * says that it caches extrainfos. (If <b>is_authority</b> we always + * believe that to be true.) */ +int +router_supports_extrainfo(const char *identity_digest, int is_authority) +{ + const node_t *node = node_get_by_id(identity_digest); + + if (node && node->ri) { + if (node->ri->caches_extra_info) + return 1; + } + if (is_authority) { + return 1; + } + return 0; +} + +/** Return true iff any trusted directory authority has accepted our + * server descriptor. + * + * We consider any authority sufficient because waiting for all of + * them means it never happens while any authority is down; we don't + * go for something more complex in the middle (like \>1/3 or \>1/2 or + * \>=1/2) because that doesn't seem necessary yet. + */ +int +directories_have_accepted_server_descriptor(void) +{ + const smartlist_t *servers = router_get_trusted_dir_servers(); + const or_options_t *options = get_options(); + SMARTLIST_FOREACH(servers, dir_server_t *, d, { + if ((d->type & options->PublishServerDescriptor_) && + d->has_accepted_serverdesc) { + return 1; + } + }); + return 0; +} + +/** Start a connection to every suitable directory authority, using + * connection purpose <b>dir_purpose</b> and uploading <b>payload</b> + * (of length <b>payload_len</b>). The dir_purpose should be one of + * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'. + * + * <b>router_purpose</b> describes the type of descriptor we're + * publishing, if we're publishing a descriptor -- e.g. general or bridge. + * + * <b>type</b> specifies what sort of dir authorities (V3, + * BRIDGE, etc) we should upload to. + * + * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of + * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b> + * bytes of <b>payload</b> hold an extra-info document. Upload the descriptor + * to all authorities, and the extra-info document to all authorities that + * support it. + */ +void +directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, + dirinfo_type_t type, + const char *payload, + size_t payload_len, size_t extrainfo_len) +{ + const or_options_t *options = get_options(); + dir_indirection_t indirection; + const smartlist_t *dirservers = router_get_trusted_dir_servers(); + int found = 0; + const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || + dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES); + tor_assert(dirservers); + /* This tries dirservers which we believe to be down, but ultimately, that's + * harmless, and we may as well err on the side of getting things uploaded. + */ + SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) { + routerstatus_t *rs = &(ds->fake_status); + size_t upload_len = payload_len; + + if ((type & ds->type) == 0) + continue; + + if (exclude_self && router_digest_is_me(ds->digest)) { + /* we don't upload to ourselves, but at least there's now at least + * one authority of this type that has what we wanted to upload. */ + found = 1; + continue; + } + + if (options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) { + log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping.", + ds->nickname, + dir_conn_purpose_to_string(dir_purpose)); + continue; + } + + found = 1; /* at least one authority of this type was listed */ + if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR) + ds->has_accepted_serverdesc = 0; + + if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) { + upload_len += extrainfo_len; + log_info(LD_DIR, "Uploading an extrainfo too (length %d)", + (int) extrainfo_len); + } + if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) { + indirection = DIRIND_ANONYMOUS; + } else if (!fascist_firewall_allows_dir_server(ds, + FIREWALL_DIR_CONNECTION, + 0)) { + if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0)) + indirection = DIRIND_ONEHOP; + else + indirection = DIRIND_ANONYMOUS; + } else { + indirection = DIRIND_DIRECT_CONN; + } + + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_payload(req, payload, upload_len); + directory_initiate_request(req); + directory_request_free(req); + } SMARTLIST_FOREACH_END(ds); + if (!found) { + char *s = authdir_type_to_string(type); + log_warn(LD_DIR, "Publishing server descriptor to directory authorities " + "of type '%s', but no authorities of that type listed!", s); + tor_free(s); + } +} + +/** Return true iff, according to the values in <b>options</b>, we should be + * using directory guards for direct downloads of directory information. */ +STATIC int +should_use_directory_guards(const or_options_t *options) +{ + /* Public (non-bridge) servers never use directory guards. */ + if (public_server_mode(options)) + return 0; + /* If guards are disabled, we can't use directory guards. + */ + if (!options->UseEntryGuards) + return 0; + /* If we're configured to fetch directory info aggressively or of a + * nonstandard type, don't use directory guards. */ + if (options->DownloadExtraInfo || options->FetchDirInfoEarly || + options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors) + return 0; + return 1; +} + +/** Pick an unconstrained directory server from among our guards, the latest + * networkstatus, or the fallback dirservers, for use in downloading + * information of type <b>type</b>, and return its routerstatus. */ +static const routerstatus_t * +directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, + uint8_t dir_purpose, + circuit_guard_state_t **guard_state_out) +{ + const routerstatus_t *rs = NULL; + const or_options_t *options = get_options(); + + if (options->UseBridges) + log_warn(LD_BUG, "Called when we have UseBridges set."); + + if (should_use_directory_guards(options)) { + const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out); + if (node) + rs = node->rs; + } else { + /* anybody with a non-zero dirport will do */ + rs = router_pick_directory_server(type, pds_flags); + } + if (!rs) { + log_info(LD_DIR, "No router found for %s; falling back to " + "dirserver list.", dir_conn_purpose_to_string(dir_purpose)); + rs = router_pick_fallback_dirserver(type, pds_flags); + } + + return rs; +} + +/** + * Set the extra fields in <b>req</b> that are used when requesting a + * consensus of type <b>resource</b>. + * + * Right now, these fields are if-modified-since and x-or-diff-from-consensus. + */ +static void +dir_consensus_request_set_additional_headers(directory_request_t *req, + const char *resource) +{ + time_t if_modified_since = 0; + uint8_t or_diff_from[DIGEST256_LEN]; + int or_diff_from_is_set = 0; + + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ + const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180; + const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72; + const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0; + const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192; + const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] = + "try-diff-for-consensus-newer-than"; + + int flav = FLAV_NS; + if (resource) + flav = networkstatus_parse_flavor_name(resource); + + int32_t max_age_for_diff = 3600 * + networkstatus_get_param(NULL, + TRY_DIFF_FOR_CONSENSUS_NEWER_NAME, + DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER, + MIN_TRY_DIFF_FOR_CONSENSUS_NEWER, + MAX_TRY_DIFF_FOR_CONSENSUS_NEWER); + + if (flav != -1) { + /* IF we have a parsed consensus of this type, we can do an + * if-modified-time based on it. */ + networkstatus_t *v; + v = networkstatus_get_latest_consensus_by_flavor(flav); + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + if (v->valid_after >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } + } else { + /* Otherwise it might be a consensus we don't parse, but which we + * do cache. Look at the cached copy, perhaps. */ + cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ + if (cd) { + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (cd->published >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } + } + } + + if (if_modified_since > 0) + directory_request_set_if_modified_since(req, if_modified_since); + if (or_diff_from_is_set) { + char hex[HEX_DIGEST256_LEN + 1]; + base16_encode(hex, sizeof(hex), + (const char*)or_diff_from, sizeof(or_diff_from)); + directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex); + } +} +/** Start a connection to a random running directory server, using + * connection purpose <b>dir_purpose</b>, intending to fetch descriptors + * of purpose <b>router_purpose</b>, and requesting <b>resource</b>. + * Use <b>pds_flags</b> as arguments to router_pick_directory_server() + * or router_pick_trusteddirserver(). + */ +MOCK_IMPL(void, +directory_get_from_dirserver,( + uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags, + download_want_authority_t want_authority)) +{ + const routerstatus_t *rs = NULL; + const or_options_t *options = get_options(); + int prefer_authority = (directory_fetches_from_authorities(options) + || want_authority == DL_WANT_AUTHORITY); + int require_authority = 0; + int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose, + resource); + dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource); + + if (type == NO_DIRINFO) + return; + + if (!options->FetchServerDescriptors) + return; + + circuit_guard_state_t *guard_state = NULL; + if (!get_via_tor) { + if (options->UseBridges && !(type & BRIDGE_DIRINFO)) { + /* We want to ask a running bridge for which we have a descriptor. + * + * When we ask choose_random_entry() for a bridge, we specify what + * sort of dir fetch we'll be doing, so it won't return a bridge + * that can't answer our question. + */ + const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state); + if (node && node->ri) { + /* every bridge has a routerinfo. */ + routerinfo_t *ri = node->ri; + /* clients always make OR connections to bridges */ + tor_addr_port_t or_ap; + directory_request_t *req = directory_request_new(dir_purpose); + /* we are willing to use a non-preferred address if we need to */ + fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, + &or_ap); + directory_request_set_or_addr_port(req, &or_ap); + directory_request_set_directory_id_digest(req, + ri->cache_info.identity_digest); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); + } else { + if (guard_state) { + entry_guard_cancel(&guard_state); + } + log_notice(LD_DIR, "Ignoring directory request, since no bridge " + "nodes are available yet."); + } + + return; + } else { + if (prefer_authority || (type & BRIDGE_DIRINFO)) { + /* only ask authdirservers, and don't ask myself */ + rs = router_pick_trusteddirserver(type, pds_flags); + if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH))) { + /* We don't want to fetch from any authorities that we're currently + * fetching server descriptors from, and we got no match. Did we + * get no match because all the authorities have connections + * fetching server descriptors (in which case we should just + * return,) or because all the authorities are down or on fire or + * unreachable or something (in which case we should go on with + * our fallback code)? */ + pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH); + rs = router_pick_trusteddirserver(type, pds_flags); + if (rs) { + log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities " + "are in use."); + return; + } + } + if (rs == NULL && require_authority) { + log_info(LD_DIR, "No authorities were available for %s: will try " + "later.", dir_conn_purpose_to_string(dir_purpose)); + return; + } + } + if (!rs && !(type & BRIDGE_DIRINFO)) { + rs = directory_pick_generic_dirserver(type, pds_flags, + dir_purpose, + &guard_state); + if (!rs) + get_via_tor = 1; /* last resort: try routing it via Tor */ + } + } + } + + if (get_via_tor) { + /* Never use fascistfirewall; we're going via Tor. */ + pds_flags |= PDS_IGNORE_FASCISTFIREWALL; + rs = router_pick_directory_server(type, pds_flags); + } + + /* If we have any hope of building an indirect conn, we know some router + * descriptors. If (rs==NULL), we can't build circuits anyway, so + * there's no point in falling back to the authorities in this case. */ + if (rs) { + const dir_indirection_t indirection = + get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_indirection(req, indirection); + directory_request_set_resource(req, resource); + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) + dir_consensus_request_set_additional_headers(req, resource); + if (guard_state) + directory_request_set_guard_state(req, guard_state); + directory_initiate_request(req); + directory_request_free(req); + } else { + log_notice(LD_DIR, + "While fetching directory info, " + "no running dirservers known. Will try again later. " + "(purpose %d)", dir_purpose); + if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) { + /* remember we tried them all and failed. */ + directory_all_unreachable(time(NULL)); + } + } +} + +/** As directory_get_from_dirserver, but initiates a request to <i>every</i> + * directory authority other than ourself. Only for use by authorities when + * searching for missing information while voting. */ +void +directory_get_from_all_authorities(uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource) +{ + tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE || + dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES); + + SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(), + dir_server_t *, ds) { + if (router_digest_is_me(ds->digest)) + continue; + if (!(ds->type & V3_DIRINFO)) + continue; + const routerstatus_t *rs = &ds->fake_status; + directory_request_t *req = directory_request_new(dir_purpose); + directory_request_set_routerstatus(req, rs); + directory_request_set_router_purpose(req, router_purpose); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); + } SMARTLIST_FOREACH_END(ds); +} + +/** Return true iff <b>ind</b> requires a multihop circuit. */ +static int +dirind_is_anon(dir_indirection_t ind) +{ + return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS; +} + +/* Choose reachable OR and Dir addresses and ports from status, copying them + * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're + * connecting via another relay, so choose the primary IPv4 address and ports. + * + * status should have at least one reachable address, if we can't choose a + * reachable address, warn and return -1. Otherwise, return 0. + */ +static int +directory_choose_address_routerstatus(const routerstatus_t *status, + dir_indirection_t indirection, + tor_addr_port_t *use_or_ap, + tor_addr_port_t *use_dir_ap) +{ + tor_assert(status != NULL); + tor_assert(use_or_ap != NULL); + tor_assert(use_dir_ap != NULL); + + const or_options_t *options = get_options(); + int have_or = 0, have_dir = 0; + + /* We expect status to have at least one reachable address if we're + * connecting to it directly. + * + * Therefore, we can simply use the other address if the one we want isn't + * allowed by the firewall. + * + * (When Tor uploads and downloads a hidden service descriptor, it uses + * DIRIND_ANONYMOUS. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, + * to avoid HSDirs denying service by rejecting descriptors.) + */ + + /* Initialise the OR / Dir addresses */ + tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC); + use_or_ap->port = 0; + tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC); + use_dir_ap->port = 0; + + /* ORPort connections */ + if (indirection == DIRIND_ANONYMOUS) { + if (status->addr) { + /* Since we're going to build a 3-hop circuit and ask the 2nd relay + * to extend to this address, always use the primary (IPv4) OR address */ + tor_addr_from_ipv4h(&use_or_ap->addr, status->addr); + use_or_ap->port = status->or_port; + have_or = 1; + } + } else if (indirection == DIRIND_ONEHOP) { + /* We use an IPv6 address if we have one and we prefer it. + * Use the preferred address and port if they are reachable, otherwise, + * use the alternate address and port (if any). + */ + fascist_firewall_choose_address_rs(status, FIREWALL_OR_CONNECTION, 0, + use_or_ap); + have_or = tor_addr_port_is_valid_ap(use_or_ap, 0); + } + + /* DirPort connections + * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */ + if (indirection == DIRIND_DIRECT_CONN || + indirection == DIRIND_ANON_DIRPORT || + (indirection == DIRIND_ONEHOP + && !directory_must_use_begindir(options))) { + fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0, + use_dir_ap); + have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0); + } + + /* We rejected all addresses in the relay's status. This means we can't + * connect to it. */ + if (!have_or && !have_dir) { + static int logged_backtrace = 0; + log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when " + "launching an outgoing directory connection to: IPv4 %s OR %d " + "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status), + fmt_addr32(status->addr), status->or_port, + status->dir_port, fmt_addr(&status->ipv6_addr), + status->ipv6_orport, status->dir_port); + if (!logged_backtrace) { + log_backtrace(LOG_INFO, LD_BUG, "Addresses came from"); + logged_backtrace = 1; + } + return -1; + } + + return 0; +} + +/** Return true iff <b>conn</b> is the client side of a directory connection + * we launched to ourself in order to determine the reachability of our + * dir_port. */ +static int +directory_conn_is_self_reachability_test(dir_connection_t *conn) +{ + if (conn->requested_resource && + !strcmpstart(conn->requested_resource,"authority")) { + const routerinfo_t *me = router_get_my_routerinfo(); + if (me && + router_digest_is_me(conn->identity_digest) && + tor_addr_eq_ipv4h(&conn->base_.addr, me->addr) && /*XXXX prop 118*/ + me->dir_port == conn->base_.port) + return 1; + } + return 0; +} + +/** Called when we are unable to complete the client's request to a directory + * server due to a network error: Mark the router as down and try again if + * possible. + */ +void +connection_dir_client_request_failed(dir_connection_t *conn) +{ + if (conn->guard_state) { + /* We haven't seen a success on this guard state, so consider it to have + * failed. */ + entry_guard_failed(&conn->guard_state); + } + if (directory_conn_is_self_reachability_test(conn)) { + return; /* this was a test fetch. don't retry. */ + } + if (!entry_list_is_constrained(get_options())) + router_set_status(conn->identity_digest, 0); /* don't try this one again */ + if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { + log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from " + "directory server at '%s'; retrying", + conn->base_.address); + if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE) + connection_dir_bridge_routerdesc_failed(conn); + connection_dir_download_routerdesc_failed(conn); + } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { + if (conn->requested_resource) + networkstatus_consensus_download_failed(0, conn->requested_resource); + } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) { + log_info(LD_DIR, "Giving up on certificate fetch from directory server " + "at '%s'; retrying", + conn->base_.address); + connection_dir_download_cert_failed(conn, 0); + } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { + log_info(LD_DIR, "Giving up downloading detached signatures from '%s'", + conn->base_.address); + } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) { + log_info(LD_DIR, "Giving up downloading votes from '%s'", + conn->base_.address); + } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) { + log_info(LD_DIR, "Giving up on downloading microdescriptors from " + "directory server at '%s'; will retry", conn->base_.address); + connection_dir_download_routerdesc_failed(conn); + } +} + +/** Helper: Attempt to fetch directly the descriptors of each bridge + * listed in <b>failed</b>. + */ +static void +connection_dir_retry_bridges(smartlist_t *descs) +{ + char digest[DIGEST_LEN]; + SMARTLIST_FOREACH(descs, const char *, cp, + { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) { + log_warn(LD_BUG, "Malformed fingerprint in list: %s", + escaped(cp)); + continue; + } + retry_bridge_descriptor_fetch_directly(digest); + }); +} + +/** Called when an attempt to download one or more router descriptors + * or extra-info documents on connection <b>conn</b> failed. + */ +static void +connection_dir_download_routerdesc_failed(dir_connection_t *conn) +{ + /* No need to increment the failure count for routerdescs, since + * it's not their fault. */ + + /* No need to relaunch descriptor downloads here: we already do it + * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || + conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + + (void) conn; +} + +/** Called when an attempt to download a bridge's routerdesc from + * one of the authorities failed due to a network error. If + * possible attempt to download descriptors from the bridge directly. + */ +static void +connection_dir_bridge_routerdesc_failed(dir_connection_t *conn) +{ + smartlist_t *which = NULL; + + /* Requests for bridge descriptors are in the form 'fp/', so ignore + anything else. */ + if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/")) + return; + + which = smartlist_new(); + dir_split_resource_into_fingerprints(conn->requested_resource + + strlen("fp/"), + which, NULL, 0); + + tor_assert(conn->base_.purpose != DIR_PURPOSE_FETCH_EXTRAINFO); + if (smartlist_len(which)) { + connection_dir_retry_bridges(which); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + } + smartlist_free(which); +} + +/** Called when an attempt to fetch a certificate fails. */ +static void +connection_dir_download_cert_failed(dir_connection_t *conn, int status) +{ + const char *fp_pfx = "fp/"; + const char *fpsk_pfx = "fp-sk/"; + smartlist_t *failed; + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE); + + if (!conn->requested_resource) + return; + failed = smartlist_new(); + /* + * We have two cases download by fingerprint (resource starts + * with "fp/") or download by fingerprint/signing key pair + * (resource starts with "fp-sk/"). + */ + if (!strcmpstart(conn->requested_resource, fp_pfx)) { + /* Download by fingerprint case */ + dir_split_resource_into_fingerprints(conn->requested_resource + + strlen(fp_pfx), + failed, NULL, DSR_HEX); + SMARTLIST_FOREACH_BEGIN(failed, char *, cp) { + /* Null signing key digest indicates download by fp only */ + authority_cert_dl_failed(cp, NULL, status); + tor_free(cp); + } SMARTLIST_FOREACH_END(cp); + } else if (!strcmpstart(conn->requested_resource, fpsk_pfx)) { + /* Download by (fp,sk) pairs */ + dir_split_resource_into_fingerprint_pairs(conn->requested_resource + + strlen(fpsk_pfx), failed); + SMARTLIST_FOREACH_BEGIN(failed, fp_pair_t *, cp) { + authority_cert_dl_failed(cp->first, cp->second, status); + tor_free(cp); + } SMARTLIST_FOREACH_END(cp); + } else { + log_warn(LD_DIR, + "Don't know what to do with failure for cert fetch %s", + conn->requested_resource); + } + + smartlist_free(failed); + + update_certificate_downloads(time(NULL)); +} + +/* Should this tor instance only use begindir for all its directory requests? + */ +int +directory_must_use_begindir(const or_options_t *options) +{ + /* Clients, onion services, and bridges must use begindir, + * relays and authorities do not have to */ + return !public_server_mode(options); +} + +/** Evaluate the situation and decide if we should use an encrypted + * "begindir-style" connection for this directory request. + * 0) If there is no DirPort, yes. + * 1) If or_port is 0, or it's a direct conn and or_port is firewalled + * or we're a dir mirror, no. + * 2) If we prefer to avoid begindir conns, and we're not fetching or + * publishing a bridge relay descriptor, no. + * 3) Else yes. + * If returning 0, return in *reason why we can't use begindir. + * reason must not be NULL. + */ +static int +directory_command_should_use_begindir(const or_options_t *options, + const directory_request_t *req, + const char **reason) +{ + const tor_addr_t *or_addr = &req->or_addr_port.addr; + //const tor_addr_t *dir_addr = &req->dir_addr_port.addr; + const int or_port = req->or_addr_port.port; + const int dir_port = req->dir_addr_port.port; + + const dir_indirection_t indirection = req->indirection; + + tor_assert(reason); + *reason = NULL; + + /* Reasons why we must use begindir */ + if (!dir_port) { + *reason = "(using begindir - directory with no DirPort)"; + return 1; /* We don't know a DirPort -- must begindir. */ + } + /* Reasons why we can't possibly use begindir */ + if (!or_port) { + *reason = "directory with unknown ORPort"; + return 0; /* We don't know an ORPort -- no chance. */ + } + if (indirection == DIRIND_DIRECT_CONN || + indirection == DIRIND_ANON_DIRPORT) { + *reason = "DirPort connection"; + return 0; + } + if (indirection == DIRIND_ONEHOP) { + /* We're firewalled and want a direct OR connection */ + if (!fascist_firewall_allows_address_addr(or_addr, or_port, + FIREWALL_OR_CONNECTION, 0, 0)) { + *reason = "ORPort not reachable"; + return 0; + } + } + /* Reasons why we want to avoid using begindir */ + if (indirection == DIRIND_ONEHOP) { + if (!directory_must_use_begindir(options)) { + *reason = "in relay mode"; + return 0; + } + } + /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS + */ + *reason = "(using begindir)"; + return 1; +} + +/** + * Create and return a new directory_request_t with purpose + * <b>dir_purpose</b>. + */ +directory_request_t * +directory_request_new(uint8_t dir_purpose) +{ + tor_assert(dir_purpose >= DIR_PURPOSE_MIN_); + tor_assert(dir_purpose <= DIR_PURPOSE_MAX_); + tor_assert(dir_purpose != DIR_PURPOSE_SERVER); + tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2); + tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_HSDESC); + + directory_request_t *result = tor_malloc_zero(sizeof(*result)); + tor_addr_make_null(&result->or_addr_port.addr, AF_INET); + result->or_addr_port.port = 0; + tor_addr_make_null(&result->dir_addr_port.addr, AF_INET); + result->dir_addr_port.port = 0; + result->dir_purpose = dir_purpose; + result->router_purpose = ROUTER_PURPOSE_GENERAL; + result->indirection = DIRIND_ONEHOP; + return result; +} +/** + * Release all resources held by <b>req</b>. + */ +void +directory_request_free_(directory_request_t *req) +{ + if (req == NULL) + return; + config_free_lines(req->additional_headers); + tor_free(req); +} +/** + * Set the address and OR port to use for this directory request. If there is + * no OR port, we'll have to connect over the dirport. (If there are both, + * the indirection setting determines which to use.) + */ +void +directory_request_set_or_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->or_addr_port, p, sizeof(*p)); +} +/** + * Set the address and dirport to use for this directory request. If there + * is no dirport, we'll have to connect over the OR port. (If there are both, + * the indirection setting determines which to use.) + */ +void +directory_request_set_dir_addr_port(directory_request_t *req, + const tor_addr_port_t *p) +{ + memcpy(&req->dir_addr_port, p, sizeof(*p)); +} +/** + * Set the RSA identity digest of the directory to use for this directory + * request. + */ +void +directory_request_set_directory_id_digest(directory_request_t *req, + const char *digest) +{ + memcpy(req->digest, digest, DIGEST_LEN); +} +/** + * Set the router purpose associated with uploaded and downloaded router + * descriptors and extrainfo documents in this directory request. The purpose + * must be one of ROUTER_PURPOSE_GENERAL (the default) or + * ROUTER_PURPOSE_BRIDGE. + */ +void +directory_request_set_router_purpose(directory_request_t *req, + uint8_t router_purpose) +{ + tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL || + router_purpose == ROUTER_PURPOSE_BRIDGE); + // assert that it actually makes sense to set this purpose, given + // the dir_purpose. + req->router_purpose = router_purpose; +} +/** + * Set the indirection to be used for the directory request. The indirection + * parameter configures whether to connect to a DirPort or ORPort, and whether + * to anonymize the connection. DIRIND_ONEHOP (use ORPort, don't anonymize) + * is the default. See dir_indirection_t for more information. + */ +void +directory_request_set_indirection(directory_request_t *req, + dir_indirection_t indirection) +{ + req->indirection = indirection; +} + +/** + * Set a pointer to the resource to request from a directory. Different + * request types use resources to indicate different components of their URL. + * Note that only an alias to <b>resource</b> is stored, so the + * <b>resource</b> must outlive the request. + */ +void +directory_request_set_resource(directory_request_t *req, + const char *resource) +{ + req->resource = resource; +} +/** + * Set a pointer to the payload to include with this directory request, along + * with its length. Note that only an alias to <b>payload</b> is stored, so + * the <b>payload</b> must outlive the request. + */ +void +directory_request_set_payload(directory_request_t *req, + const char *payload, + size_t payload_len) +{ + tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose)); + + req->payload = payload; + req->payload_len = payload_len; +} +/** + * Set an if-modified-since date to send along with the request. The + * default is 0 (meaning, send no if-modified-since header). + */ +void +directory_request_set_if_modified_since(directory_request_t *req, + time_t if_modified_since) +{ + req->if_modified_since = if_modified_since; +} + +/** Include a header of name <b>key</b> with content <b>val</b> in the + * request. Neither may include newlines or other odd characters. Their + * ordering is not currently guaranteed. + * + * Note that, as elsewhere in this module, header keys include a trailing + * colon and space. + */ +void +directory_request_add_header(directory_request_t *req, + const char *key, + const char *val) +{ + config_line_prepend(&req->additional_headers, key, val); +} +/** + * Set an object containing HS data to be associated with this request. Note + * that only an alias to <b>query</b> is stored, so the <b>query</b> object + * must outlive the request. + */ +void +directory_request_set_rend_query(directory_request_t *req, + const rend_data_t *query) +{ + if (query) { + tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 || + req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); + } + req->rend_query = query; +} +/** + * Set an object containing HS connection identifier to be associated with + * this request. Note that only an alias to <b>ident</b> is stored, so the + * <b>ident</b> object must outlive the request. + */ +void +directory_request_upload_set_hs_ident(directory_request_t *req, + const hs_ident_dir_conn_t *ident) +{ + if (ident) { + tor_assert(req->dir_purpose == DIR_PURPOSE_UPLOAD_HSDESC); + } + req->hs_ident = ident; +} +/** + * Set an object containing HS connection identifier to be associated with + * this fetch request. Note that only an alias to <b>ident</b> is stored, so + * the <b>ident</b> object must outlive the request. + */ +void +directory_request_fetch_set_hs_ident(directory_request_t *req, + const hs_ident_dir_conn_t *ident) +{ + if (ident) { + tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_HSDESC); + } + req->hs_ident = ident; +} +/** Set a static circuit_guard_state_t object to affliate with the request in + * <b>req</b>. This object will receive notification when the attempt to + * connect to the guard either succeeds or fails. */ +void +directory_request_set_guard_state(directory_request_t *req, + circuit_guard_state_t *state) +{ + req->guard_state = state; +} + +/** + * Internal: Return true if any information for contacting the directory in + * <b>req</b> has been set, other than by the routerstatus. */ +static int +directory_request_dir_contact_info_specified(const directory_request_t *req) +{ + /* We only check for ports here, since we don't use an addr unless the port + * is set */ + return (req->or_addr_port.port || + req->dir_addr_port.port || + ! tor_digest_is_zero(req->digest)); +} + +/** + * Set the routerstatus to use for the directory associated with this + * request. If this option is set, then no other function to set the + * directory's address or identity should be called. + */ +void +directory_request_set_routerstatus(directory_request_t *req, + const routerstatus_t *status) +{ + req->routerstatus = status; +} +/** + * Helper: update the addresses, ports, and identities in <b>req</b> + * from the routerstatus object in <b>req</b>. Return 0 on success. + * On failure, warn and return -1. + */ +static int +directory_request_set_dir_from_routerstatus(directory_request_t *req) + +{ + const routerstatus_t *status = req->routerstatus; + if (BUG(status == NULL)) + return -1; + const or_options_t *options = get_options(); + const node_t *node; + tor_addr_port_t use_or_ap, use_dir_ap; + const int anonymized_connection = dirind_is_anon(req->indirection); + + tor_assert(status != NULL); + + node = node_get_by_id(status->identity_digest); + + /* XXX The below check is wrong: !node means it's not in the consensus, + * but we haven't checked if we have a descriptor for it -- and also, + * we only care about the descriptor if it's a begindir-style anonymized + * connection. */ + if (!node && anonymized_connection) { + log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " + "don't have its router descriptor.", + routerstatus_describe(status)); + return -1; + } + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { + log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping. This choice might make your Tor not work.", + routerstatus_describe(status), + dir_conn_purpose_to_string(req->dir_purpose)); + return -1; + } + + /* At this point, if we are a client making a direct connection to a + * directory server, we have selected a server that has at least one address + * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This + * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if + * possible. (If UseBridges is set, clients always use IPv6, and prefer it + * by default.) + * + * Now choose an address that we can use to connect to the directory server. + */ + if (directory_choose_address_routerstatus(status, + req->indirection, &use_or_ap, + &use_dir_ap) < 0) { + return -1; + } + + directory_request_set_or_addr_port(req, &use_or_ap); + directory_request_set_dir_addr_port(req, &use_dir_ap); + directory_request_set_directory_id_digest(req, status->identity_digest); + return 0; +} + +/** + * Launch the provided directory request, configured in <b>request</b>. + * After this function is called, you can free <b>request</b>. + */ +MOCK_IMPL(void, +directory_initiate_request,(directory_request_t *request)) +{ + tor_assert(request); + if (request->routerstatus) { + tor_assert_nonfatal( + ! directory_request_dir_contact_info_specified(request)); + if (directory_request_set_dir_from_routerstatus(request) < 0) { + return; + } + } + + const tor_addr_port_t *or_addr_port = &request->or_addr_port; + const tor_addr_port_t *dir_addr_port = &request->dir_addr_port; + const char *digest = request->digest; + const uint8_t dir_purpose = request->dir_purpose; + const uint8_t router_purpose = request->router_purpose; + const dir_indirection_t indirection = request->indirection; + const char *resource = request->resource; + const rend_data_t *rend_query = request->rend_query; + const hs_ident_dir_conn_t *hs_ident = request->hs_ident; + circuit_guard_state_t *guard_state = request->guard_state; + + tor_assert(or_addr_port->port || dir_addr_port->port); + tor_assert(digest); + + dir_connection_t *conn; + const or_options_t *options = get_options(); + int socket_error = 0; + const char *begindir_reason = NULL; + /* Should the connection be to a relay's OR port (and inside that we will + * send our directory request)? */ + const int use_begindir = + directory_command_should_use_begindir(options, request, &begindir_reason); + + /* Will the connection go via a three-hop Tor circuit? Note that this + * is separate from whether it will use_begindir. */ + const int anonymized_connection = dirind_is_anon(indirection); + + /* What is the address we want to make the directory request to? If + * we're making a begindir request this is the ORPort of the relay + * we're contacting; if not a begindir request, this is its DirPort. + * Note that if anonymized_connection is true, we won't be initiating + * a connection directly to this address. */ + tor_addr_t addr; + tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr); + uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port; + + log_debug(LD_DIR, "anonymized %d, use_begindir %d.", + anonymized_connection, use_begindir); + + log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + + if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) { + tor_assert(anonymized_connection || + rend_non_anonymous_mode_enabled(options)); + } + + /* use encrypted begindir connections for everything except relays + * this provides better protection for directory fetches */ + if (!use_begindir && directory_must_use_begindir(options)) { + log_warn(LD_BUG, "Client could not use begindir connection: %s", + begindir_reason ? begindir_reason : "(NULL)"); + return; + } + + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HTTPProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + + /* Make sure that the destination addr and port we picked is viable. */ + if (!port || tor_addr_is_null(&addr)) { + static int logged_backtrace = 0; + log_warn(LD_DIR, + "Cannot make an outgoing %sconnection without a remote %sPort.", + use_begindir ? "begindir " : "", + use_begindir ? "OR" : "Dir"); + if (!logged_backtrace) { + log_backtrace(LOG_INFO, LD_BUG, "Address came from"); + logged_backtrace = 1; + } + return; + } + + conn = dir_connection_new(tor_addr_family(&addr)); + + /* set up conn so it's got all the data we need to remember */ + tor_addr_copy(&conn->base_.addr, &addr); + conn->base_.port = port; + conn->base_.address = tor_addr_to_str_dup(&addr); + memcpy(conn->identity_digest, digest, DIGEST_LEN); + + conn->base_.purpose = dir_purpose; + conn->router_purpose = router_purpose; + + /* give it an initial state */ + conn->base_.state = DIR_CONN_STATE_CONNECTING; + + /* decide whether we can learn our IP address from this conn */ + /* XXXX This is a bad name for this field now. */ + conn->dirconn_direct = !anonymized_connection; + + /* copy rendezvous data, if any */ + if (rend_query) { + /* We can't have both v2 and v3+ identifier. */ + tor_assert_nonfatal(!hs_ident); + conn->rend_data = rend_data_dup(rend_query); + } + if (hs_ident) { + /* We can't have both v2 and v3+ identifier. */ + tor_assert_nonfatal(!rend_query); + conn->hs_ident = hs_ident_dir_conn_dup(hs_ident); + } + + if (!anonymized_connection && !use_begindir) { + /* then we want to connect to dirport directly */ + + if (options->HTTPProxy) { + tor_addr_copy(&addr, &options->HTTPProxyAddr); + port = options->HTTPProxyPort; + } + + // In this case we should not have picked a directory guard. + if (BUG(guard_state)) { + entry_guard_cancel(&guard_state); + } + + switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, + port, &socket_error)) { + case -1: + connection_mark_for_close(TO_CONN(conn)); + return; + case 1: + /* start flushing conn */ + conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; + /* fall through */ + case 0: + /* queue the command on the outbuf */ + directory_send_command(conn, 1, request); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); + /* writable indicates finish, readable indicates broken link, + error indicates broken link in windowsland. */ + } + } else { + /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with + * begindir, maybe not with begindir) */ + + entry_connection_t *linked_conn; + + /* Anonymized tunneled connections can never share a circuit. + * One-hop directory connections can share circuits with each other + * but nothing else. */ + int iso_flags = anonymized_connection ? ISO_STREAM : ISO_SESSIONGRP; + + /* If it's an anonymized connection, remember the fact that we + * wanted it for later: maybe we'll want it again soon. */ + if (anonymized_connection && use_begindir) + rep_hist_note_used_internal(time(NULL), 0, 1); + else if (anonymized_connection && !use_begindir) + rep_hist_note_used_port(time(NULL), conn->base_.port); + + // In this case we should not have a directory guard; we'll + // get a regular guard later when we build the circuit. + if (BUG(anonymized_connection && guard_state)) { + entry_guard_cancel(&guard_state); + } + + conn->guard_state = guard_state; + + /* make an AP connection + * populate it and add it at the right state + * hook up both sides + */ + linked_conn = + connection_ap_make_link(TO_CONN(conn), + conn->base_.address, conn->base_.port, + digest, + SESSION_GROUP_DIRCONN, iso_flags, + use_begindir, !anonymized_connection); + if (!linked_conn) { + log_warn(LD_NET,"Making tunnel to dirserver failed."); + connection_mark_for_close(TO_CONN(conn)); + return; + } + + if (connection_add(TO_CONN(conn)) < 0) { + log_warn(LD_NET,"Unable to add connection for link to dirserver."); + connection_mark_for_close(TO_CONN(conn)); + return; + } + conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; + /* queue the command on the outbuf */ + directory_send_command(conn, 0, request); + + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); + connection_start_reading(ENTRY_TO_CONN(linked_conn)); + } +} + +/** Helper for sorting + * + * sort strings alphabetically + * + * XXXX we have a smartlist_sort_strings() function, right? + */ +static int +compare_strs_(const void **a, const void **b) +{ + const char *s1 = *a, *s2 = *b; + return strcmp(s1, s2); +} + +#define CONDITIONAL_CONSENSUS_FPR_LEN 3 +#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN) +#error "conditional consensus fingerprint length is larger than digest length" +#endif + +/** Return the URL we should use for a consensus download. + * + * Use the "conditional consensus downloading" feature described in + * dir-spec.txt, i.e. + * GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b> + * + * If 'resource' is provided, it is the name of a consensus flavor to request. + */ +static char * +directory_get_consensus_url(const char *resource) +{ + char *url = NULL; + const char *hyphen, *flavor; + if (resource==NULL || strcmp(resource, "ns")==0) { + flavor = ""; /* Request ns consensuses as "", so older servers will work*/ + hyphen = ""; + } else { + flavor = resource; + hyphen = "-"; + } + + { + char *authority_id_list; + smartlist_t *authority_digests = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(), + dir_server_t *, ds) { + char *hex; + if (!(ds->type & V3_DIRINFO)) + continue; + + hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); + base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, + ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); + smartlist_add(authority_digests, hex); + } SMARTLIST_FOREACH_END(ds); + smartlist_sort(authority_digests, compare_strs_); + authority_id_list = smartlist_join_strings(authority_digests, + "+", 0, NULL); + + tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z", + hyphen, flavor, authority_id_list); + + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); + tor_free(authority_id_list); + } + return url; +} + +/** + * Copies the ipv6 from source to destination, subject to buffer size limit + * size. If decorate is true, makes sure the copied address is decorated. + */ +static void +copy_ipv6_address(char* destination, const char* source, size_t len, + int decorate) { + tor_assert(destination); + tor_assert(source); + + if (decorate && source[0] != '[') { + tor_snprintf(destination, len, "[%s]", source); + } else { + strlcpy(destination, source, len); + } +} + +/** Queue an appropriate HTTP command for <b>request</b> on + * <b>conn</b>-\>outbuf. If <b>direct</b> is true, we're making a + * non-anonymized connection to the dirport. + */ +static void +directory_send_command(dir_connection_t *conn, + const int direct, + const directory_request_t *req) +{ + tor_assert(req); + const int purpose = req->dir_purpose; + const char *resource = req->resource; + const char *payload = req->payload; + const size_t payload_len = req->payload_len; + const time_t if_modified_since = req->if_modified_since; + const int anonymized_connection = dirind_is_anon(req->indirection); + + char proxystring[256]; + char hoststring[128]; + /* NEEDS to be the same size hoststring. + Will be decorated with brackets around it if it is ipv6. */ + char decorated_address[128]; + smartlist_t *headers = smartlist_new(); + char *url; + char *accept_encoding; + size_t url_len; + char request[8192]; + size_t request_len, total_request_len = 0; + const char *httpcommand = NULL; + + tor_assert(conn); + tor_assert(conn->base_.type == CONN_TYPE_DIR); + + tor_free(conn->requested_resource); + if (resource) + conn->requested_resource = tor_strdup(resource); + + /* decorate the ip address if it is ipv6 */ + if (strchr(conn->base_.address, ':')) { + copy_ipv6_address(decorated_address, conn->base_.address, + sizeof(decorated_address), 1); + } else { + strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address)); + } + + /* come up with a string for which Host: we want */ + if (conn->base_.port == 80) { + strlcpy(hoststring, decorated_address, sizeof(hoststring)); + } else { + tor_snprintf(hoststring, sizeof(hoststring), "%s:%d", + decorated_address, conn->base_.port); + } + + /* Format if-modified-since */ + if (if_modified_since) { + char b[RFC1123_TIME_LEN+1]; + format_rfc1123_time(b, if_modified_since); + smartlist_add_asprintf(headers, "If-Modified-Since: %s\r\n", b); + } + + /* come up with some proxy lines, if we're using one. */ + if (direct && get_options()->HTTPProxy) { + char *base64_authenticator=NULL; + const char *authenticator = get_options()->HTTPProxyAuthenticator; + + tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring); + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_BUG, "Encoding http authenticator failed"); + } + if (base64_authenticator) { + smartlist_add_asprintf(headers, + "Proxy-Authorization: Basic %s\r\n", + base64_authenticator); + tor_free(base64_authenticator); + } + } else { + proxystring[0] = 0; + } + + if (! anonymized_connection) { + /* Add Accept-Encoding. */ + accept_encoding = accept_encoding_header(); + smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n", + accept_encoding); + tor_free(accept_encoding); + } + + /* Add additional headers, if any */ + { + config_line_t *h; + for (h = req->additional_headers; h; h = h->next) { + smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value); + } + } + + switch (purpose) { + case DIR_PURPOSE_FETCH_CONSENSUS: + /* resource is optional. If present, it's a flavor name */ + tor_assert(!payload); + httpcommand = "GET"; + url = directory_get_consensus_url(resource); + log_info(LD_DIR, "Downloading consensus from %s using %s", + hoststring, url); + break; + case DIR_PURPOSE_FETCH_CERTIFICATE: + tor_assert(resource); + tor_assert(!payload); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/keys/%s", resource); + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + tor_assert(resource); + tor_assert(!payload); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/status-vote/next/%s.z", resource); + break; + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + tor_assert(!resource); + tor_assert(!payload); + httpcommand = "GET"; + url = tor_strdup("/tor/status-vote/next/consensus-signatures.z"); + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + tor_assert(resource); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/server/%s", resource); + break; + case DIR_PURPOSE_FETCH_EXTRAINFO: + tor_assert(resource); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/extra/%s", resource); + break; + case DIR_PURPOSE_FETCH_MICRODESC: + tor_assert(resource); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/micro/%s", resource); + break; + case DIR_PURPOSE_UPLOAD_DIR: { + const char *why = router_get_descriptor_gen_reason(); + tor_assert(!resource); + tor_assert(payload); + httpcommand = "POST"; + url = tor_strdup("/tor/"); + if (!why) { + why = "for no reason at all"; + } + smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why); + break; + } + case DIR_PURPOSE_UPLOAD_VOTE: + tor_assert(!resource); + tor_assert(payload); + httpcommand = "POST"; + url = tor_strdup("/tor/post/vote"); + break; + case DIR_PURPOSE_UPLOAD_SIGNATURES: + tor_assert(!resource); + tor_assert(payload); + httpcommand = "POST"; + url = tor_strdup("/tor/post/consensus-signature"); + break; + case DIR_PURPOSE_FETCH_RENDDESC_V2: + tor_assert(resource); + tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); + tor_assert(!payload); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/rendezvous2/%s", resource); + break; + case DIR_PURPOSE_FETCH_HSDESC: + tor_assert(resource); + tor_assert(strlen(resource) <= ED25519_BASE64_LEN); + tor_assert(!payload); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/hs/3/%s", resource); + break; + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + tor_assert(!resource); + tor_assert(payload); + httpcommand = "POST"; + url = tor_strdup("/tor/rendezvous2/publish"); + break; + case DIR_PURPOSE_UPLOAD_HSDESC: + tor_assert(resource); + tor_assert(payload); + httpcommand = "POST"; + tor_asprintf(&url, "/tor/hs/%s/publish", resource); + break; + default: + tor_assert(0); + return; + } + + /* warn in the non-tunneled case */ + if (direct && (strlen(proxystring) + strlen(url) >= 4096)) { + log_warn(LD_BUG, + "Squid does not like URLs longer than 4095 bytes, and this " + "one is %d bytes long: %s%s", + (int)(strlen(proxystring) + strlen(url)), proxystring, url); + } + + tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring); + + request_len = strlen(request); + total_request_len += request_len; + connection_buf_add(request, request_len, TO_CONN(conn)); + + url_len = strlen(url); + total_request_len += url_len; + connection_buf_add(url, url_len, TO_CONN(conn)); + tor_free(url); + + if (!strcmp(httpcommand, "POST") || payload) { + smartlist_add_asprintf(headers, "Content-Length: %lu\r\n", + payload ? (unsigned long)payload_len : 0); + } + + { + char *header = smartlist_join_strings(headers, "", 0, NULL); + tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n", + hoststring, header); + tor_free(header); + } + + request_len = strlen(request); + total_request_len += request_len; + connection_buf_add(request, request_len, TO_CONN(conn)); + + if (payload) { + /* then send the payload afterwards too */ + connection_buf_add(payload, payload_len, TO_CONN(conn)); + total_request_len += payload_len; + } + + SMARTLIST_FOREACH(headers, char *, h, tor_free(h)); + smartlist_free(headers); + + log_debug(LD_DIR, + "Sent request to directory server '%s:%d': " + "(purpose: %d, request size: %"TOR_PRIuSZ", " + "payload size: %"TOR_PRIuSZ")", + conn->base_.address, conn->base_.port, + conn->base_.purpose, + (total_request_len), + (payload ? payload_len : 0)); +} + +/** Return true iff <b>body</b> doesn't start with a plausible router or + * network-status or microdescriptor opening. This is a sign of possible + * compression. */ +static int +body_is_plausible(const char *body, size_t len, int purpose) +{ + int i; + if (len == 0) + return 1; /* empty bodies don't need decompression */ + if (len < 32) + return 0; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + return (!strcmpstart(body,"onion-key")); + } + + if (!strcmpstart(body,"router") || + !strcmpstart(body,"network-status")) + return 1; + for (i=0;i<32;++i) { + if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i])) + return 0; + } + + return 1; +} + +/** Called when we've just fetched a bunch of router descriptors in + * <b>body</b>. The list <b>which</b>, if present, holds digests for + * descriptors we requested: descriptor digests if <b>descriptor_digests</b> + * is true, or identity digests otherwise. Parse the descriptors, validate + * them, and annotate them as having purpose <b>purpose</b> and as having been + * downloaded from <b>source</b>. + * + * Return the number of routers actually added. */ +static int +load_downloaded_routers(const char *body, smartlist_t *which, + int descriptor_digests, + int router_purpose, + const char *source) +{ + char buf[256]; + char time_buf[ISO_TIME_LEN+1]; + int added = 0; + int general = router_purpose == ROUTER_PURPOSE_GENERAL; + format_iso_time(time_buf, time(NULL)); + tor_assert(source); + + if (tor_snprintf(buf, sizeof(buf), + "@downloaded-at %s\n" + "@source %s\n" + "%s%s%s", time_buf, escaped(source), + !general ? "@purpose " : "", + !general ? router_purpose_to_string(router_purpose) : "", + !general ? "\n" : "")<0) + return added; + + added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, + descriptor_digests, buf); + if (added && general) + control_event_boot_dir(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); + return added; +} + +static int handle_response_fetch_certificate(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_status_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_detached_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_desc(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_dir(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_vote(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_signatures(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_fetch_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_renddesc_v2(dir_connection_t *, + const response_handler_args_t *); +static int handle_response_upload_hsdesc(dir_connection_t *, + const response_handler_args_t *); + +static int +dir_client_decompress_response_body(char **bodyp, size_t *bodylenp, + dir_connection_t *conn, + compress_method_t compression, + int anonymized_connection) +{ + int rv = 0; + const char *body = *bodyp; + size_t body_len = *bodylenp; + int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || + conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + + int plausible = body_is_plausible(body, body_len, conn->base_.purpose); + + if (plausible && compression == NO_METHOD) { + return 0; + } + + int severity = LOG_DEBUG; + char *new_body = NULL; + size_t new_len = 0; + const char *description1, *description2; + int want_to_try_both = 0; + int tried_both = 0; + compress_method_t guessed = detect_compression_method(body, body_len); + + description1 = compression_method_get_human_name(compression); + + if (BUG(description1 == NULL)) + description1 = compression_method_get_human_name(UNKNOWN_METHOD); + + if (guessed == UNKNOWN_METHOD && !plausible) + description2 = "confusing binary junk"; + else + description2 = compression_method_get_human_name(guessed); + + /* Tell the user if we don't believe what we're told about compression.*/ + want_to_try_both = (compression == UNKNOWN_METHOD || + guessed != compression); + if (want_to_try_both) { + severity = LOG_PROTOCOL_WARN; + } + + tor_log(severity, LD_HTTP, + "HTTP body from server '%s:%d' was labeled as %s, " + "%s it seems to be %s.%s", + conn->base_.address, conn->base_.port, description1, + guessed != compression?"but":"and", + description2, + (compression>0 && guessed>0 && want_to_try_both)? + " Trying both.":""); + + /* Try declared compression first if we can. + * tor_compress_supports_method() also returns true for NO_METHOD. + * Ensure that the server is not sending us data compressed using a + * compression method that is not allowed for anonymous connections. */ + if (anonymized_connection && + ! allowed_anonymous_connection_compression_method(compression)) { + warn_disallowed_anonymous_compression_method(compression); + rv = -1; + goto done; + } + + if (tor_compress_supports_method(compression)) { + tor_uncompress(&new_body, &new_len, body, body_len, compression, + !allow_partial, LOG_PROTOCOL_WARN); + if (new_body) { + /* We succeeded with the declared compression method. Great! */ + rv = 0; + goto done; + } + } + + /* Okay, if that didn't work, and we think that it was compressed + * differently, try that. */ + if (anonymized_connection && + ! allowed_anonymous_connection_compression_method(guessed)) { + warn_disallowed_anonymous_compression_method(guessed); + rv = -1; + goto done; + } + + if (tor_compress_supports_method(guessed) && + compression != guessed) { + tor_uncompress(&new_body, &new_len, body, body_len, guessed, + !allow_partial, LOG_INFO); + tried_both = 1; + } + /* If we're pretty sure that we have a compressed directory, and + * we didn't manage to uncompress it, then warn and bail. */ + if (!plausible && !new_body) { + log_fn(LOG_PROTOCOL_WARN, LD_HTTP, + "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').", + description1, + tried_both?" and ":"", + tried_both?description2:"", + conn->base_.address, conn->base_.port); + rv = -1; + goto done; + } + + done: + if (new_body) { + if (rv == 0) { + /* success! */ + tor_free(*bodyp); + *bodyp = new_body; + *bodylenp = new_len; + } else { + tor_free(new_body); + } + } + + return rv; +} + +/** We are a client, and we've finished reading the server's + * response. Parse it and act appropriately. + * + * If we're still happy with using this directory server in the future, return + * 0. Otherwise return -1; and the caller should consider trying the request + * again. + * + * The caller will take care of marking the connection for close. + */ +static int +connection_dir_client_reached_eof(dir_connection_t *conn) +{ + char *body = NULL; + char *headers = NULL; + char *reason = NULL; + size_t body_len = 0; + int status_code; + time_t date_header = 0; + long apparent_skew; + compress_method_t compression; + int skewed = 0; + int rv; + int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || + conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + size_t received_bytes; + const int anonymized_connection = + purpose_needs_anonymity(conn->base_.purpose, + conn->router_purpose, + conn->requested_resource); + + received_bytes = connection_get_inbuf_len(TO_CONN(conn)); + + switch (connection_fetch_from_buf_http(TO_CONN(conn), + &headers, MAX_HEADERS_SIZE, + &body, &body_len, MAX_DIR_DL_SIZE, + allow_partial)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "'fetch' response too large (server '%s:%d'). Closing.", + conn->base_.address, conn->base_.port); + return -1; + case 0: + log_info(LD_HTTP, + "'fetch' response not all here, but we're at eof. Closing."); + return -1; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + &compression, &reason) < 0) { + log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.", + conn->base_.address, conn->base_.port); + + rv = -1; + goto done; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + tor_log(LOG_DEBUG, LD_DIR, + "Received response from directory server '%s:%d': %d %s " + "(purpose: %d, response size: %"TOR_PRIuSZ +#ifdef MEASUREMENTS_21206 + ", data cells received: %d, data cells sent: %d" +#endif + ", compression: %d)", + conn->base_.address, conn->base_.port, status_code, + escaped(reason), conn->base_.purpose, + (received_bytes), +#ifdef MEASUREMENTS_21206 + conn->data_cells_received, conn->data_cells_sent, +#endif + compression); + + if (conn->guard_state) { + /* we count the connection as successful once we can read from it. We do + * not, however, delay use of the circuit here, since it's just for a + * one-hop directory request. */ + /* XXXXprop271 note that this will not do the right thing for other + * waiting circuits that would be triggered by this circuit becoming + * complete/usable. But that's ok, I think. + */ + entry_guard_succeeded(&conn->guard_state); + circuit_guard_state_free(conn->guard_state); + conn->guard_state = NULL; + } + + /* now check if it's got any hints for us about our IP address. */ + if (conn->dirconn_direct) { + char *guess = http_get_header(headers, X_ADDRESS_HEADER); + if (guess) { + router_new_address_suggestion(guess, conn); + tor_free(guess); + } + } + + if (date_header > 0) { + /* The date header was written very soon after we sent our request, + * so compute the skew as the difference between sending the request + * and the date header. (We used to check now-date_header, but that's + * inaccurate if we spend a lot of time downloading.) + */ + apparent_skew = conn->base_.timestamp_last_write_allowed - date_header; + if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) { + int trusted = router_digest_is_trusted_dir(conn->identity_digest); + clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP, + "directory", "DIRSERV"); + skewed = 1; /* don't check the recommended-versions line */ + } else { + log_debug(LD_HTTP, "Time on received directory is within tolerance; " + "we are %ld seconds skewed. (That's okay.)", apparent_skew); + } + } + (void) skewed; /* skewed isn't used yet. */ + + if (status_code == 503) { + routerstatus_t *rs; + dir_server_t *ds; + const char *id_digest = conn->identity_digest; + log_info(LD_DIR,"Received http status code %d (%s) from server " + "'%s:%d'. I'll try again soon.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + time_t now = approx_time(); + if ((rs = router_get_mutable_consensus_status_by_id(id_digest))) + rs->last_dir_503_at = now; + if ((ds = router_get_fallback_dirserver_by_digest(id_digest))) + ds->fake_status.last_dir_503_at = now; + + rv = -1; + goto done; + } + + if (dir_client_decompress_response_body(&body, &body_len, + conn, compression, anonymized_connection) < 0) { + rv = -1; + goto done; + } + + response_handler_args_t args; + memset(&args, 0, sizeof(args)); + args.status_code = status_code; + args.reason = reason; + args.body = body; + args.body_len = body_len; + args.headers = headers; + + switch (conn->base_.purpose) { + case DIR_PURPOSE_FETCH_CONSENSUS: + rv = handle_response_fetch_consensus(conn, &args); + break; + case DIR_PURPOSE_FETCH_CERTIFICATE: + rv = handle_response_fetch_certificate(conn, &args); + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + rv = handle_response_fetch_status_vote(conn, &args); + break; + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + rv = handle_response_fetch_detached_signatures(conn, &args); + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + rv = handle_response_fetch_desc(conn, &args); + break; + case DIR_PURPOSE_FETCH_MICRODESC: + rv = handle_response_fetch_microdesc(conn, &args); + break; + case DIR_PURPOSE_FETCH_RENDDESC_V2: + rv = handle_response_fetch_renddesc_v2(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_DIR: + rv = handle_response_upload_dir(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_SIGNATURES: + rv = handle_response_upload_signatures(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_VOTE: + rv = handle_response_upload_vote(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + rv = handle_response_upload_renddesc_v2(conn, &args); + break; + case DIR_PURPOSE_UPLOAD_HSDESC: + rv = handle_response_upload_hsdesc(conn, &args); + break; + case DIR_PURPOSE_FETCH_HSDESC: + rv = handle_response_fetch_hsdesc_v3(conn, &args); + break; + default: + tor_assert_nonfatal_unreached(); + rv = -1; + break; + } + + done: + tor_free(body); + tor_free(headers); + tor_free(reason); + return rv; +} + +/** + * Handler function: processes a response to a request for a networkstatus + * consensus document by checking the consensus, storing it, and marking + * router requests as reachable. + **/ +STATIC int +handle_response_fetch_consensus(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS); + const int status_code = args->status_code; + const char *body = args->body; + const size_t body_len = args->body_len; + const char *reason = args->reason; + const time_t now = approx_time(); + + const char *consensus; + char *new_consensus = NULL; + const char *sourcename; + + int r; + const char *flavname = conn->requested_resource; + if (status_code != 200) { + int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; + tor_log(severity, LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching consensus directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + networkstatus_consensus_download_failed(status_code, flavname); + return -1; + } + + if (looks_like_a_consensus_diff(body, body_len)) { + /* First find our previous consensus. Maybe it's in ram, maybe not. */ + cached_dir_t *cd = dirserv_get_consensus(flavname); + const char *consensus_body; + char *owned_consensus = NULL; + if (cd) { + consensus_body = cd->dir; + } else { + owned_consensus = networkstatus_read_cached_consensus(flavname); + consensus_body = owned_consensus; + } + if (!consensus_body) { + log_warn(LD_DIR, "Received a consensus diff, but we can't find " + "any %s-flavored consensus in our current cache.",flavname); + networkstatus_consensus_download_failed(0, flavname); + // XXXX if this happens too much, see below + return -1; + } + + new_consensus = consensus_diff_apply(consensus_body, body); + tor_free(owned_consensus); + if (new_consensus == NULL) { + log_warn(LD_DIR, "Could not apply consensus diff received from server " + "'%s:%d'", conn->base_.address, conn->base_.port); + // XXXX If this happens too many times, we should maybe not use + // XXXX this directory for diffs any more? + networkstatus_consensus_download_failed(0, flavname); + return -1; + } + log_info(LD_DIR, "Applied consensus diff (size %d) from server " + "'%s:%d', resulting in a new consensus document (size %d).", + (int)body_len, conn->base_.address, conn->base_.port, + (int)strlen(new_consensus)); + consensus = new_consensus; + sourcename = "generated based on a diff"; + } else { + log_info(LD_DIR,"Received consensus directory (body size %d) from server " + "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); + consensus = body; + sourcename = "downloaded"; + } + + if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + conn->identity_digest))<0) { + log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, + "Unable to load %s consensus directory %s from " + "server '%s:%d'. I'll try again soon.", + flavname, sourcename, conn->base_.address, conn->base_.port); + networkstatus_consensus_download_failed(0, flavname); + tor_free(new_consensus); + return -1; + } + + /* If we launched other fetches for this consensus, cancel them. */ + connection_dir_close_consensus_fetches(conn, flavname); + + /* update the list of routers and directory guards */ + routers_update_all_from_networkstatus(now, 3); + update_microdescs_from_networkstatus(now); + directory_info_has_arrived(now, 0, 0); + + if (authdir_mode_v3(get_options())) { + sr_act_post_consensus( + networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); + } + log_info(LD_DIR, "Successfully loaded consensus."); + + tor_free(new_consensus); + return 0; +} + +/** + * Handler function: processes a response to a request for one or more + * authority certificates + **/ +static int +handle_response_fetch_certificate(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + if (status_code != 200) { + log_warn(LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/keys/%s\".", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); + return -1; + } + log_info(LD_DIR,"Received authority certificates (body size %d) from " + "server '%s:%d'", + (int)body_len, conn->base_.address, conn->base_.port); + + /* + * Tell trusted_dirs_load_certs_from_string() whether it was by fp + * or fp-sk pair. + */ + int src_code = -1; + if (!strcmpstart(conn->requested_resource, "fp/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST; + } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) { + src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST; + } + + if (src_code != -1) { + if (trusted_dirs_load_certs_from_string(body, src_code, 1, + conn->identity_digest)<0) { + log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ + connection_dir_download_cert_failed(conn, status_code); + } else { + time_t now = approx_time(); + directory_info_has_arrived(now, 0, 0); + log_info(LD_DIR, "Successfully loaded certificates from fetch."); + } + } else { + log_warn(LD_DIR, + "Couldn't figure out what to do with fetched certificates for " + "unknown resource %s", + conn->requested_resource); + connection_dir_download_cert_failed(conn, status_code); + } + return 0; +} + +/** + * Handler function: processes a response to a request for an authority's + * current networkstatus vote. + **/ +static int +handle_response_fetch_status_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg; + int st; + log_info(LD_DIR,"Got votes (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + return -1; + } + dirvote_add_vote(body, &msg, &st); + if (st > 299) { + log_warn(LD_DIR, "Error adding retrieved vote: %s", msg); + } else { + log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg); + } + + return 0; +} + +/** + * Handler function: processes a response to a request for the signatures + * that an authority knows about on a given consensus. + **/ +static int +handle_response_fetch_detached_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + const char *msg = NULL; + log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d", + (int)body_len, conn->base_.address, conn->base_.port); + if (status_code != 200) { + log_warn(LD_DIR, + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + return -1; + } + if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) { + log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s", + conn->base_.address, conn->base_.port, msg?msg:"???"); + } + + return 0; +} + +/** + * Handler function: processes a response to a request for a group of server + * descriptors or an extrainfo documents. + **/ +static int +handle_response_fetch_desc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO; + smartlist_t *which = NULL; + int n_asked_for = 0; + int descriptor_digests = conn->requested_resource && + !strcmpstart(conn->requested_resource,"d/"); + log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'", + was_ei ? "extra server info" : "server info", + (int)body_len, conn->base_.address, conn->base_.port); + if (conn->requested_resource && + (!strcmpstart(conn->requested_resource,"d/") || + !strcmpstart(conn->requested_resource,"fp/"))) { + which = smartlist_new(); + dir_split_resource_into_fingerprints(conn->requested_resource + + (descriptor_digests ? 2 : 3), + which, NULL, 0); + n_asked_for = smartlist_len(which); + } + if (status_code != 200) { + int dir_okay = status_code == 404 || + (status_code == 400 && !strcmp(reason, "Servers unavailable.")); + /* 404 means that it didn't have them; no big deal. + * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */ + log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR, + "Received http status code %d (%s) from server '%s:%d' " + "while fetching \"/tor/server/%s\". I'll try again soon.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port, conn->requested_resource); + if (!which) { + connection_dir_download_routerdesc_failed(conn); + } else { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + } + return dir_okay ? 0 : -1; + } + /* Learn the routers, assuming we requested by fingerprint or "all" + * or "authority". + * + * We use "authority" to fetch our own descriptor for + * testing, and to fetch bridge descriptors for bootstrapping. Ignore + * the output of "authority" requests unless we are using bridges, + * since otherwise they'll be the response from reachability tests, + * and we don't really want to add that to our routerlist. */ + if (which || (conn->requested_resource && + (!strcmpstart(conn->requested_resource, "all") || + (!strcmpstart(conn->requested_resource, "authority") && + get_options()->UseBridges)))) { + /* as we learn from them, we remove them from 'which' */ + if (was_ei) { + router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which, + descriptor_digests); + } else { + //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, + // descriptor_digests, conn->router_purpose); + if (load_downloaded_routers(body, which, descriptor_digests, + conn->router_purpose, + conn->base_.address)) { + time_t now = approx_time(); + directory_info_has_arrived(now, 0, 1); + } + } + } + if (which) { /* mark remaining ones as failed */ + log_info(LD_DIR, "Received %d/%d %s requested from %s:%d", + n_asked_for-smartlist_len(which), n_asked_for, + was_ei ? "extra-info documents" : "router descriptors", + conn->base_.address, (int)conn->base_.port); + if (smartlist_len(which)) { + dir_routerdesc_download_failed(which, status_code, + conn->router_purpose, + was_ei, descriptor_digests); + } + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + } + if (directory_conn_is_self_reachability_test(conn)) + router_dirport_found_reachable(); + + return 0; +} + +/** + * Handler function: processes a response to a request for a group of + * microdescriptors + **/ +STATIC int +handle_response_fetch_microdesc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + smartlist_t *which = NULL; + log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " + "body size %d) from server '%s:%d'", + status_code, (int)body_len, conn->base_.address, + conn->base_.port); + tor_assert(conn->requested_resource && + !strcmpstart(conn->requested_resource, "d/")); + tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN)); + which = smartlist_new(); + dir_split_resource_into_fingerprints(conn->requested_resource+2, + which, NULL, + DSR_DIGEST256|DSR_BASE64); + if (status_code != 200) { + log_info(LD_DIR, "Received status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " + "soon.", + status_code, escaped(reason), conn->base_.address, + (int)conn->base_.port, conn->requested_resource); + dir_microdesc_download_failed(which, status_code, conn->identity_digest); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + return 0; + } else { + smartlist_t *mds; + time_t now = approx_time(); + mds = microdescs_add_to_cache(get_microdesc_cache(), + body, body+body_len, SAVED_NOWHERE, 0, + now, which); + if (smartlist_len(which)) { + /* Mark remaining ones as failed. */ + dir_microdesc_download_failed(which, status_code, conn->identity_digest); + } + if (mds && smartlist_len(mds)) { + control_event_boot_dir(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); + directory_info_has_arrived(now, 0, 1); + } + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + smartlist_free(mds); + } + + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload our + * router descriptor. + **/ +static int +handle_response_upload_dir(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *headers = args->headers; + + switch (status_code) { + case 200: { + dir_server_t *ds = + router_get_trusteddirserver_by_digest(conn->identity_digest); + char *rejected_hdr = http_get_header(headers, + "X-Descriptor-Not-New: "); + if (rejected_hdr) { + if (!strcmp(rejected_hdr, "Yes")) { + log_info(LD_GENERAL, + "Authority '%s' declined our descriptor (not new)", + ds->nickname); + /* XXXX use this information; be sure to upload next one + * sooner. -NM */ + /* XXXX++ On further thought, the task above implies that we're + * basing our regenerate-descriptor time on when we uploaded the + * last descriptor, not on the published time of the last + * descriptor. If those are different, that's a bad thing to + * do. -NM */ + } + tor_free(rejected_hdr); + } + log_info(LD_GENERAL,"eof (status 200) after uploading server " + "descriptor: finished."); + control_event_server_status( + LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d", + conn->base_.address, conn->base_.port); + + ds->has_accepted_serverdesc = 1; + if (directories_have_accepted_server_descriptor()) + control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR"); + } + break; + case 400: + log_warn(LD_GENERAL,"http status 400 (%s) response from " + "dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + control_event_server_status(LOG_WARN, + "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"", + conn->base_.address, conn->base_.port, escaped(reason)); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "descriptor to server '%s:%d'. Possibly the server is " + "misconfigured?", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + + return 0; +} + +/** + * Handler function: processes a response to POST request to upload our + * own networkstatus vote. + **/ +static int +handle_response_upload_vote(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "vote to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "vote to server '%s:%d'.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + return 0; +} + +/** + * Handler function: processes a response to POST request to upload our + * view of the signatures on the current consensus. + **/ +static int +handle_response_upload_signatures(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES); + const int status_code = args->status_code; + const char *reason = args->reason; + + switch (status_code) { + case 200: { + log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d", + conn->base_.address, conn->base_.port); + } + break; + case 400: + log_warn(LD_DIR,"http status 400 (%s) response after uploading " + "signatures to dirserver '%s:%d'. Please correct.", + escaped(reason), conn->base_.address, conn->base_.port); + break; + default: + log_warn(LD_GENERAL, + "HTTP status %d (%s) was unexpected while uploading " + "signatures to server '%s:%d'.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + break; + } + /* return 0 in all cases, since we don't want to mark any + * dirservers down just because they don't like us. */ + + return 0; +} + +/** + * Handler function: processes a response to a request for a v3 hidden service + * descriptor. + **/ +STATIC int +handle_response_fetch_hsdesc_v3(dir_connection_t *conn, + const response_handler_args_t *args) +{ + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + + tor_assert(conn->hs_ident); + + log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))", + (int)body_len, status_code, escaped(reason)); + + switch (status_code) { + case 200: + /* We got something: Try storing it in the cache. */ + if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) { + log_info(LD_REND, "Failed to store hidden service descriptor"); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "BAD_DESC"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + } else { + log_info(LD_REND, "Stored hidden service descriptor successfully."); + TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC; + hs_client_desc_has_arrived(conn->hs_ident); + /* Fire control port RECEIVED event. */ + hs_control_desc_event_received(conn->hs_ident, conn->identity_digest); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + body); + } + break; + case 404: + /* Not there. We'll retry when connection_about_to_close_connection() + * tries to clean this conn up. */ + log_info(LD_REND, "Fetching hidden service v3 descriptor not found: " + "Retrying at another directory."); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "NOT_FOUND"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + break; + case 400: + log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: " + "http status 400 (%s). Dirserver didn't like our " + "query? Retrying at another directory.", + escaped(reason)); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "QUERY_REJECTED"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + break; + default: + log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: " + "http status %d (%s) response unexpected from HSDir server " + "'%s:%d'. Retrying at another directory.", + status_code, escaped(reason), TO_CONN(conn)->address, + TO_CONN(conn)->port); + /* Fire control port FAILED event. */ + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "UNEXPECTED"); + hs_control_desc_event_content(conn->hs_ident, conn->identity_digest, + NULL); + break; + } + + return 0; +} + +/** + * Handler function: processes a response to a request for a v2 hidden service + * descriptor. + **/ +static int +handle_response_fetch_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + const char *body = args->body; + const size_t body_len = args->body_len; + +#define SEND_HS_DESC_FAILED_EVENT(reason) \ + (control_event_hsv2_descriptor_failed(conn->rend_data, \ + conn->identity_digest, \ + reason)) +#define SEND_HS_DESC_FAILED_CONTENT() \ + (control_event_hs_descriptor_content( \ + rend_data_get_address(conn->rend_data), \ + conn->requested_resource, \ + conn->identity_digest, \ + NULL)) + + tor_assert(conn->rend_data); + log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d " + "(%s))", + (int)body_len, status_code, escaped(reason)); + switch (status_code) { + case 200: + { + rend_cache_entry_t *entry = NULL; + + if (rend_cache_store_v2_desc_as_client(body, + conn->requested_resource, + conn->rend_data, &entry) < 0) { + log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " + "Retrying at another directory."); + /* We'll retry when connection_about_to_close_connection() + * cleans this dir conn up. */ + SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); + SEND_HS_DESC_FAILED_CONTENT(); + } else { + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + /* Should never be NULL here if we found the descriptor. */ + tor_assert(entry); + rend_get_service_id(entry->parsed->pk, service_id); + + /* success. notify pending connections about this. */ + log_info(LD_REND, "Successfully fetched v2 rendezvous " + "descriptor."); + control_event_hsv2_descriptor_received(service_id, + conn->rend_data, + conn->identity_digest); + control_event_hs_descriptor_content(service_id, + conn->requested_resource, + conn->identity_digest, + body); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; + rend_client_desc_trynow(service_id); + memwipe(service_id, 0, sizeof(service_id)); + } + break; + } + case 404: + /* Not there. We'll retry when + * connection_about_to_close_connection() cleans this conn up. */ + log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " + "Retrying at another directory."); + SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + case 400: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status 400 (%s). Dirserver didn't like our " + "v2 rendezvous query? Retrying at another directory.", + escaped(reason)); + SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + default: + log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " + "http status %d (%s) response unexpected while " + "fetching v2 hidden service descriptor (server '%s:%d'). " + "Retrying at another directory.", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + SEND_HS_DESC_FAILED_CONTENT(); + break; + } + + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload a v2 + * hidden service descriptor. + **/ +static int +handle_response_upload_renddesc_v2(dir_connection_t *conn, + const response_handler_args_t *args) +{ + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2); + const int status_code = args->status_code; + const char *reason = args->reason; + +#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) \ + (control_event_hs_descriptor_upload_failed( \ + conn->identity_digest, \ + rend_data_get_address(conn->rend_data), \ + reason)) + + log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " + "(%s))", + status_code, escaped(reason)); + /* Without the rend data, we'll have a problem identifying what has been + * uploaded for which service. */ + tor_assert(conn->rend_data); + switch (status_code) { + case 200: + log_info(LD_REND, + "Uploading rendezvous descriptor: finished with status " + "200 (%s)", escaped(reason)); + control_event_hs_descriptor_uploaded(conn->identity_digest, + rend_data_get_address(conn->rend_data)); + rend_service_desc_has_uploaded(conn->rend_data); + break; + case 400: + log_warn(LD_REND,"http status 400 (%s) response from dirserver " + "'%s:%d'. Malformed rendezvous descriptor?", + escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); + break; + default: + log_warn(LD_REND,"http status %d (%s) response unexpected (server " + "'%s:%d').", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); + break; + } + + return 0; +} + +/** + * Handler function: processes a response to a POST request to upload an + * hidden service descriptor. + **/ +static int +handle_response_upload_hsdesc(dir_connection_t *conn, + const response_handler_args_t *args) +{ + const int status_code = args->status_code; + const char *reason = args->reason; + + tor_assert(conn); + tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_HSDESC); + + log_info(LD_REND, "Uploaded hidden service descriptor (status %d " + "(%s))", + status_code, escaped(reason)); + /* For this directory response, it MUST have an hidden service identifier on + * this connection. */ + tor_assert(conn->hs_ident); + switch (status_code) { + case 200: + log_info(LD_REND, "Uploading hidden service descriptor: " + "finished with status 200 (%s)", escaped(reason)); + hs_control_desc_event_uploaded(conn->hs_ident, conn->identity_digest); + break; + case 400: + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Uploading hidden service descriptor: http " + "status 400 (%s) response from dirserver " + "'%s:%d'. Malformed hidden service descriptor?", + escaped(reason), conn->base_.address, conn->base_.port); + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "UPLOAD_REJECTED"); + break; + default: + log_warn(LD_REND, "Uploading hidden service descriptor: http " + "status %d (%s) response unexpected (server " + "'%s:%d').", + status_code, escaped(reason), conn->base_.address, + conn->base_.port); + hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest, + "UNEXPECTED"); + break; + } + + return 0; +} + +/** Called when a directory connection reaches EOF. */ +int +connection_dir_reached_eof(dir_connection_t *conn) +{ + int retval; + if (conn->base_.state != DIR_CONN_STATE_CLIENT_READING) { + log_info(LD_HTTP,"conn reached eof, not reading. [state=%d] Closing.", + conn->base_.state); + connection_close_immediate(TO_CONN(conn)); /* error: give up on flushing */ + connection_mark_for_close(TO_CONN(conn)); + return -1; + } + + retval = connection_dir_client_reached_eof(conn); + if (retval == 0) /* success */ + conn->base_.state = DIR_CONN_STATE_CLIENT_FINISHED; + connection_mark_for_close(TO_CONN(conn)); + return retval; +} +/** We are closing a dir connection: If <b>dir_conn</b> is a dir connection + * that tried to fetch an HS descriptor, check if it successfully fetched it, + * or if we need to try again. */ +void +connection_dir_client_refetch_hsdesc_if_needed(dir_connection_t *dir_conn) +{ + connection_t *conn = TO_CONN(dir_conn); + + /* If we were trying to fetch a v2 rend desc and did not succeed, retry as + * needed. (If a fetch is successful, the connection state is changed to + * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to + * mark that refetching is unnecessary.) */ + if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 && + dir_conn->rend_data && + rend_valid_v2_service_id( + rend_data_get_address(dir_conn->rend_data))) { + rend_client_refetch_v2_renddesc(dir_conn->rend_data); + } + + /* Check for v3 rend desc fetch */ + if (conn->purpose == DIR_PURPOSE_FETCH_HSDESC && + dir_conn->hs_ident && + !ed25519_public_key_is_zero(&dir_conn->hs_ident->identity_pk)) { + hs_client_refetch_hsdesc(&dir_conn->hs_ident->identity_pk); + } +} + +/** Array of compression methods to use (if supported) for requesting + * compressed data, ordered from best to worst. */ +static compress_method_t client_meth_pref[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Array of allowed compression methods to use (if supported) when receiving a + * response from a request that was required to be anonymous. */ +static compress_method_t client_meth_allowed_anonymous_compression[] = { + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Return a newly allocated string containing a comma separated list of + * supported encodings. */ +STATIC char * +accept_encoding_header(void) +{ + smartlist_t *methods = smartlist_new(); + char *header = NULL; + compress_method_t method; + unsigned i; + + for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { + method = client_meth_pref[i]; + if (tor_compress_supports_method(method)) + smartlist_add(methods, (char *)compression_method_get_name(method)); + } + + header = smartlist_join_strings(methods, ", ", 0, NULL); + smartlist_free(methods); + + return header; +} + +/** Check if the given compression method is allowed for a connection that is + * supposed to be anonymous. Returns 1 if the compression method is allowed, + * otherwise 0. */ +STATIC int +allowed_anonymous_connection_compression_method(compress_method_t method) +{ + unsigned u; + + for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression); + ++u) { + compress_method_t allowed_method = + client_meth_allowed_anonymous_compression[u]; + + if (! tor_compress_supports_method(allowed_method)) + continue; + + if (method == allowed_method) + return 1; + } + + return 0; +} + +/** Log a warning when a remote server has sent us a document using a + * compression method that is not allowed for anonymous directory requests. */ +STATIC void +warn_disallowed_anonymous_compression_method(compress_method_t method) +{ + log_fn(LOG_PROTOCOL_WARN, LD_HTTP, + "Received a %s HTTP response, which is not " + "allowed for anonymous directory requests.", + compression_method_get_human_name(method)); +} + +/* We just got a new consensus! If there are other in-progress requests + * for this consensus flavor (for example because we launched several in + * parallel), cancel them. + * + * We do this check here (not just in + * connection_ap_handshake_attach_circuit()) to handle the edge case where + * a consensus fetch begins and ends before some other one tries to attach to + * a circuit, in which case the other one won't know that we're all happy now. + * + * Don't mark the conn that just gave us the consensus -- otherwise we + * would end up double-marking it when it cleans itself up. + */ +static void +connection_dir_close_consensus_fetches(dir_connection_t *except_this_one, + const char *resource) +{ + smartlist_t *conns_to_close = + connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS, + resource); + SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) { + if (d == except_this_one) + continue; + log_info(LD_DIR, "Closing consensus fetch (to %s) since one " + "has just arrived.", TO_CONN(d)->address); + connection_mark_for_close(TO_CONN(d)); + } SMARTLIST_FOREACH_END(d); + smartlist_free(conns_to_close); +} +/** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) + * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, + * either as descriptor digests or as identity digests based on + * <b>was_descriptor_digests</b>). + */ +static void +dir_routerdesc_download_failed(smartlist_t *failed, int status_code, + int router_purpose, + int was_extrainfo, int was_descriptor_digests) +{ + char digest[DIGEST_LEN]; + time_t now = time(NULL); + int server = directory_fetches_from_authorities(get_options()); + if (!was_descriptor_digests) { + if (router_purpose == ROUTER_PURPOSE_BRIDGE) { + tor_assert(!was_extrainfo); + connection_dir_retry_bridges(failed); + } + return; /* FFFF should implement for other-than-router-purpose someday */ + } + SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) { + download_status_t *dls = NULL; + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) { + log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp)); + continue; + } + if (was_extrainfo) { + signed_descriptor_t *sd = + router_get_by_extrainfo_digest(digest); + if (sd) + dls = &sd->ei_dl_status; + } else { + dls = router_get_dl_status_by_descriptor_digest(digest); + } + if (!dls) + continue; + download_status_increment_failure(dls, status_code, cp, server, now); + } SMARTLIST_FOREACH_END(cp); + + /* No need to relaunch descriptor downloads here: we already do it + * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */ +} + +/** Called when a connection to download microdescriptors from relay with + * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list + * of every microdesc digest we didn't get. <b>status_code</b> is the http + * status code we received. Reschedule the microdesc downloads as + * appropriate. */ +static void +dir_microdesc_download_failed(smartlist_t *failed, + int status_code, const char *dir_id) +{ + networkstatus_t *consensus + = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC); + routerstatus_t *rs; + download_status_t *dls; + time_t now = time(NULL); + int server = directory_fetches_from_authorities(get_options()); + + if (! consensus) + return; + + /* We failed to fetch a microdescriptor from 'dir_id', note it down + * so that we don't try the same relay next time... */ + microdesc_note_outdated_dirserver(dir_id); + + SMARTLIST_FOREACH_BEGIN(failed, const char *, d) { + rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d); + if (!rs) + continue; + dls = &rs->dl_status; + + { /* Increment the failure count for this md fetch */ + char buf[BASE64_DIGEST256_LEN+1]; + digest256_to_base64(buf, d); + log_info(LD_DIR, "Failed to download md %s from %s", + buf, hex_str(dir_id, DIGEST_LEN)); + download_status_increment_failure(dls, status_code, buf, + server, now); + } + } SMARTLIST_FOREACH_END(d); +} |