diff options
Diffstat (limited to 'src/or/directory.c')
| -rw-r--r-- | src/or/directory.c | 1063 |
1 files changed, 789 insertions, 274 deletions
diff --git a/src/or/directory.c b/src/or/directory.c index 50863d0c7e..89b08223d2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1,9 +1,10 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" +#include "backtrace.h" #include "buffers.h" #include "circuitbuild.h" #include "config.h" @@ -20,8 +21,10 @@ #include "networkstatus.h" #include "nodelist.h" #include "policies.h" +#include "relay.h" #include "rendclient.h" #include "rendcommon.h" +#include "rendservice.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@ -63,8 +66,6 @@ static void directory_send_command(dir_connection_t *conn, time_t if_modified_since); static int directory_handle_command(dir_connection_t *conn); static int body_is_plausible(const char *body, size_t body_len, int purpose); -static int purpose_needs_anonymity(uint8_t dir_purpose, - uint8_t router_purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); @@ -82,18 +83,21 @@ static void dir_microdesc_download_failed(smartlist_t *failed, static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); -static void directory_initiate_command_rend(const tor_addr_t *addr, - uint16_t or_port, - uint16_t dir_port, - const char *digest, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query); +static void directory_initiate_command_rend( + const tor_addr_port_t *or_addr_port, + const tor_addr_port_t *dir_addr_port, + const char *digest, + uint8_t dir_purpose, + uint8_t router_purpose, + dir_indirection_t indirection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since, + const rend_data_t *rend_query); + +static void connection_dir_close_consensus_fetches( + dir_connection_t *except_this_one, const char *resource); /********* START VARIABLES **********/ @@ -119,7 +123,7 @@ static void directory_initiate_command_rend(const tor_addr_t *addr, /** Return true iff the directory purpose <b>dir_purpose</b> (and if it's * fetching descriptors, it's fetching them for <b>router_purpose</b>) * must use an anonymous connection to a directory. */ -static int +STATIC int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { if (get_options()->AllDirActionsPrivate) @@ -143,7 +147,7 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) /** Return a newly allocated string describing <b>auth</b>. Only describes * authority features. */ -static char * +STATIC char * authdir_type_to_string(dirinfo_type_t auth) { char *result; @@ -162,7 +166,7 @@ authdir_type_to_string(dirinfo_type_t auth) } /** Return a string describing a given directory connection purpose. */ -static const char * +STATIC const char * dir_conn_purpose_to_string(int purpose) { switch (purpose) @@ -197,9 +201,49 @@ dir_conn_purpose_to_string(int purpose) return "(unknown)"; } -/** Return true iff <b>identity_digest</b> is the digest of a router we - * believe to support extrainfo downloads. (If <b>is_authority</b> we do - * additional checking that's only valid for authorities.) */ +/** Return the requisite directory information types. */ +STATIC dirinfo_type_t +dir_fetch_type(int dir_purpose, int router_purpose, const char *resource) +{ + dirinfo_type_t type; + switch (dir_purpose) { + case DIR_PURPOSE_FETCH_EXTRAINFO: + type = EXTRAINFO_DIRINFO; + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type |= BRIDGE_DIRINFO; + else + type |= V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type = BRIDGE_DIRINFO; + else + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_CERTIFICATE: + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_CONSENSUS: + type = V3_DIRINFO; + if (resource && !strcmp(resource, "microdesc")) + type |= MICRODESC_DIRINFO; + break; + case DIR_PURPOSE_FETCH_MICRODESC: + type = MICRODESC_DIRINFO; + break; + default: + log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); + type = NO_DIRINFO; + break; + } + return type; +} + +/** Return true iff <b>identity_digest</b> is the digest of a router which + * says that it caches extrainfos. (If <b>is_authority</b> we always + * believe that to be true.) */ int router_supports_extrainfo(const char *identity_digest, int is_authority) { @@ -273,7 +317,6 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) { routerstatus_t *rs = &(ds->fake_status); size_t upload_len = payload_len; - tor_addr_t ds_addr; if ((type & ds->type) == 0) continue; @@ -304,11 +347,12 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, log_info(LD_DIR, "Uploading an extrainfo too (length %d)", (int) extrainfo_len); } - tor_addr_from_ipv4h(&ds_addr, ds->addr); if (purpose_needs_anonymity(dir_purpose, router_purpose)) { indirection = DIRIND_ANONYMOUS; - } else if (!fascist_firewall_allows_address_dir(&ds_addr,ds->dir_port)) { - if (fascist_firewall_allows_address_or(&ds_addr,ds->or_port)) + } else if (!fascist_firewall_allows_dir_server(ds, + FIREWALL_DIR_CONNECTION, + 0)) { + if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0)) indirection = DIRIND_ONEHOP; else indirection = DIRIND_ANONYMOUS; @@ -330,7 +374,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, /** Return true iff, according to the values in <b>options</b>, we should be * using directory guards for direct downloads of directory information. */ -static int +STATIC int should_use_directory_guards(const or_options_t *options) { /* Public (non-bridge) servers never use directory guards. */ @@ -385,47 +429,24 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, * Use <b>pds_flags</b> as arguments to router_pick_directory_server() * or router_pick_trusteddirserver(). */ -void -directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, int pds_flags) +MOCK_IMPL(void, directory_get_from_dirserver, ( + uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags, + download_want_authority_t want_authority)) { const routerstatus_t *rs = NULL; const or_options_t *options = get_options(); - int prefer_authority = directory_fetches_from_authorities(options); + int prefer_authority = (directory_fetches_from_authorities(options) + || want_authority == DL_WANT_AUTHORITY); int require_authority = 0; int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose); - dirinfo_type_t type; + dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource); time_t if_modified_since = 0; - /* FFFF we could break this switch into its own function, and call - * it elsewhere in directory.c. -RD */ - switch (dir_purpose) { - case DIR_PURPOSE_FETCH_EXTRAINFO: - type = EXTRAINFO_DIRINFO | - (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : - V3_DIRINFO); - break; - case DIR_PURPOSE_FETCH_SERVERDESC: - type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : - V3_DIRINFO); - break; - case DIR_PURPOSE_FETCH_STATUS_VOTE: - case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: - case DIR_PURPOSE_FETCH_CERTIFICATE: - type = V3_DIRINFO; - break; - case DIR_PURPOSE_FETCH_CONSENSUS: - type = V3_DIRINFO; - if (resource && !strcmp(resource,"microdesc")) - type |= MICRODESC_DIRINFO; - break; - case DIR_PURPOSE_FETCH_MICRODESC: - type = MICRODESC_DIRINFO; - break; - default: - log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); - return; - } + if (type == NO_DIRINFO) + return; if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { int flav = FLAV_NS; @@ -433,18 +454,33 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (resource) flav = networkstatus_parse_flavor_name(resource); + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ +#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) if (flav != -1) { /* IF we have a parsed consensus of this type, we can do an * if-modified-time based on it. */ v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) - if_modified_since = v->valid_after + 180; + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + } } else { /* Otherwise it might be a consensus we don't parse, but which we * do cache. Look at the cached copy, perhaps. */ cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ if (cd) - if_modified_since = cd->published + 180; + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; } } @@ -452,7 +488,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, return; if (!get_via_tor) { - if (options->UseBridges && type != BRIDGE_DIRINFO) { + if (options->UseBridges && !(type & BRIDGE_DIRINFO)) { /* We want to ask a running bridge for which we have a descriptor. * * When we ask choose_random_entry() for a bridge, we specify what @@ -464,11 +500,14 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, const node_t *node = choose_random_dirguard(type); if (node && node->ri) { /* every bridge has a routerinfo. */ - tor_addr_t addr; routerinfo_t *ri = node->ri; - node_get_addr(node, &addr); - directory_initiate_command(&addr, - ri->or_port, 0/*no dirport*/, + /* clients always make OR connections to bridges */ + tor_addr_port_t or_ap; + /* we are willing to use a non-preferred address if we need to */ + fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, + &or_ap); + directory_initiate_command(&or_ap.addr, or_ap.port, + NULL, 0, /*no dirport*/ ri->cache_info.identity_digest, dir_purpose, router_purpose, @@ -479,7 +518,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, "nodes are available yet."); return; } else { - if (prefer_authority || type == BRIDGE_DIRINFO) { + if (prefer_authority || (type & BRIDGE_DIRINFO)) { /* only ask authdirservers, and don't ask myself */ rs = router_pick_trusteddirserver(type, pds_flags); if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| @@ -506,29 +545,25 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, return; } } - if (!rs && type != BRIDGE_DIRINFO) { + if (!rs && !(type & BRIDGE_DIRINFO)) { /* */ rs = directory_pick_generic_dirserver(type, pds_flags, dir_purpose); - if (!rs) { - /*XXXX024 I'm pretty sure this can never do any good, since - * rs isn't set. */ + if (!rs) get_via_tor = 1; /* last resort: try routing it via Tor */ - } } } - } else { /* get_via_tor */ + } + + if (get_via_tor) { /* Never use fascistfirewall; we're going via Tor. */ - if (1) { - /* anybody with a non-zero dirport will do. Disregard firewalls. */ - pds_flags |= PDS_IGNORE_FASCISTFIREWALL; - rs = router_pick_directory_server(type, pds_flags); - /* If we have any hope of building an indirect conn, we know some router - * descriptors. If (rs==NULL), we can't build circuits anyway, so - * there's no point in falling back to the authorities in this case. */ - } + pds_flags |= PDS_IGNORE_FASCISTFIREWALL; + rs = router_pick_directory_server(type, pds_flags); } + /* If we have any hope of building an indirect conn, we know some router + * descriptors. If (rs==NULL), we can't build circuits anyway, so + * there's no point in falling back to the authorities in this case. */ if (rs) { const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; @@ -581,6 +616,95 @@ dirind_is_anon(dir_indirection_t ind) return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS; } +/* Choose reachable OR and Dir addresses and ports from status, copying them + * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're + * connecting via another relay, so choose the primary IPv4 address and ports. + * + * status should have at least one reachable address, if we can't choose a + * reachable address, warn and return -1. Otherwise, return 0. + */ +static int +directory_choose_address_routerstatus(const routerstatus_t *status, + dir_indirection_t indirection, + tor_addr_port_t *use_or_ap, + tor_addr_port_t *use_dir_ap) +{ + tor_assert(status != NULL); + tor_assert(use_or_ap != NULL); + tor_assert(use_dir_ap != NULL); + + const or_options_t *options = get_options(); + int have_or = 0, have_dir = 0; + + /* We expect status to have at least one reachable address if we're + * connecting to it directly. + * + * Therefore, we can simply use the other address if the one we want isn't + * allowed by the firewall. + * + * (When Tor uploads and downloads a hidden service descriptor, it uses + * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP. + * So this code will only modify the address for Tor2Web's HS descriptor + * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid + * HSDirs denying service by rejecting descriptors.) + */ + + /* Initialise the OR / Dir addresses */ + tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC); + use_or_ap->port = 0; + tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC); + use_dir_ap->port = 0; + + /* ORPort connections */ + if (indirection == DIRIND_ANONYMOUS) { + if (status->addr) { + /* Since we're going to build a 3-hop circuit and ask the 2nd relay + * to extend to this address, always use the primary (IPv4) OR address */ + tor_addr_from_ipv4h(&use_or_ap->addr, status->addr); + use_or_ap->port = status->or_port; + have_or = 1; + } + } else if (indirection == DIRIND_ONEHOP) { + /* We use an IPv6 address if we have one and we prefer it. + * Use the preferred address and port if they are reachable, otherwise, + * use the alternate address and port (if any). + */ + have_or = fascist_firewall_choose_address_rs(status, + FIREWALL_OR_CONNECTION, 0, + use_or_ap); + } + + /* DirPort connections + * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */ + if (indirection == DIRIND_DIRECT_CONN || + indirection == DIRIND_ANON_DIRPORT || + (indirection == DIRIND_ONEHOP + && !directory_must_use_begindir(options))) { + have_dir = fascist_firewall_choose_address_rs(status, + FIREWALL_DIR_CONNECTION, 0, + use_dir_ap); + } + + /* We rejected all addresses in the relay's status. This means we can't + * connect to it. */ + if (!have_or && !have_dir) { + static int logged_backtrace = 0; + log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when " + "launching an outgoing directory connection to: IPv4 %s OR %d " + "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status), + fmt_addr32(status->addr), status->or_port, + status->dir_port, fmt_addr(&status->ipv6_addr), + status->ipv6_orport, status->dir_port); + if (!logged_backtrace) { + log_backtrace(LOG_INFO, LD_BUG, "Addresses came from"); + logged_backtrace = 1; + } + return -1; + } + + return 0; +} + /** Same as directory_initiate_command_routerstatus(), but accepts * rendezvous data to fetch a hidden service descriptor. */ void @@ -596,8 +720,11 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, { const or_options_t *options = get_options(); const node_t *node; - tor_addr_t addr; + tor_addr_port_t use_or_ap, use_dir_ap; const int anonymized_connection = dirind_is_anon(indirection); + + tor_assert(status != NULL); + node = node_get_by_id(status->identity_digest); if (!node && anonymized_connection) { @@ -606,7 +733,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, routerstatus_describe(status)); return; } - tor_addr_from_ipv4h(&addr, status->addr); if (options->ExcludeNodes && options->StrictNodes && routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { @@ -618,13 +744,30 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - directory_initiate_command_rend(&addr, - status->or_port, status->dir_port, - status->identity_digest, - dir_purpose, router_purpose, - indirection, resource, - payload, payload_len, if_modified_since, - rend_query); + /* At this point, if we are a clients making a direct connection to a + * directory server, we have selected a server that has at least one address + * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This + * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if + * possible. (If UseBridges is set, clients always use IPv6, and prefer it + * by default.) + * + * Now choose an address that we can use to connect to the directory server. + */ + if (directory_choose_address_routerstatus(status, indirection, &use_or_ap, + &use_dir_ap) < 0) { + return; + } + + /* We don't retry the alternate OR/Dir address for the same directory if + * the address we choose fails (#6772). + * Instead, we'll retry another directory on failure. */ + + directory_initiate_command_rend(&use_or_ap, &use_dir_ap, + status->identity_digest, + dir_purpose, router_purpose, + indirection, resource, + payload, payload_len, if_modified_since, + rend_query); } /** Launch a new connection to the directory server <b>status</b> to @@ -641,15 +784,15 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we * want to fetch. */ -void -directory_initiate_command_routerstatus(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since) +MOCK_IMPL(void, directory_initiate_command_routerstatus, + (const routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + dir_indirection_t indirection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since)) { directory_initiate_command_routerstatus_rend(status, dir_purpose, router_purpose, @@ -687,7 +830,7 @@ connection_dir_request_failed(dir_connection_t *conn) return; /* this was a test fetch. don't retry. */ } if (!entry_list_is_constrained(get_options())) - router_set_status(conn->identity_digest, 0); /* don't try him again */ + router_set_status(conn->identity_digest, 0); /* don't try this one again */ if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from " @@ -826,6 +969,16 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) update_certificate_downloads(time(NULL)); } +/* Should this tor instance only use begindir for all its directory requests? + */ +int +directory_must_use_begindir(const or_options_t *options) +{ + /* Clients, onion services, and bridges must use begindir, + * relays and authorities do not have to */ + return !public_server_mode(options); +} + /** Evaluate the situation and decide if we should use an encrypted * "begindir-style" connection for this directory request. * 1) If or_port is 0, or it's a direct conn and or_port is firewalled @@ -833,40 +986,90 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 2) If we prefer to avoid begindir conns, and we're not fetching or * publishing a bridge relay descriptor, no. * 3) Else yes. + * If returning 0, return in *reason why we can't use begindir. + * reason must not be NULL. */ static int directory_command_should_use_begindir(const or_options_t *options, const tor_addr_t *addr, int or_port, uint8_t router_purpose, - dir_indirection_t indirection) + dir_indirection_t indirection, + const char **reason) { (void) router_purpose; - if (!or_port) + tor_assert(reason); + *reason = NULL; + + /* Reasons why we can't possibly use begindir */ + if (!or_port) { + *reason = "directory with unknown ORPort"; return 0; /* We don't know an ORPort -- no chance. */ - if (indirection == DIRIND_DIRECT_CONN || indirection == DIRIND_ANON_DIRPORT) + } + if (indirection == DIRIND_DIRECT_CONN || + indirection == DIRIND_ANON_DIRPORT) { + *reason = "DirPort connection"; return 0; - if (indirection == DIRIND_ONEHOP) - if (!fascist_firewall_allows_address_or(addr, or_port) || - directory_fetches_from_authorities(options)) - return 0; /* We're firewalled or are acting like a relay -- also no. */ + } + if (indirection == DIRIND_ONEHOP) { + /* We're firewalled and want a direct OR connection */ + if (!fascist_firewall_allows_address_addr(addr, or_port, + FIREWALL_OR_CONNECTION, 0, 0)) { + *reason = "ORPort not reachable"; + return 0; + } + } + /* Reasons why we want to avoid using begindir */ + if (indirection == DIRIND_ONEHOP) { + if (!directory_must_use_begindir(options)) { + *reason = "in relay mode"; + return 0; + } + } + /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS + */ + *reason = "(using begindir)"; return 1; } -/** Helper for directory_initiate_command_routerstatus: send the - * command to a server whose address is <b>address</b>, whose IP is - * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version - * <b>supports_begindir</b>, and whose identity key digest is - * <b>digest</b>. */ +/** Helper for directory_initiate_command_rend: send the + * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>, + * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose + * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and + * <b>router_purpose</b>, making an (in)direct connection as specified in + * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of + * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>. + */ void -directory_initiate_command(const tor_addr_t *_addr, - uint16_t or_port, uint16_t dir_port, +directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, + const tor_addr_t *dir_addr, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, dir_indirection_t indirection, const char *resource, const char *payload, size_t payload_len, time_t if_modified_since) { - directory_initiate_command_rend(_addr, or_port, dir_port, + tor_addr_port_t or_ap, dir_ap; + + /* Use the null tor_addr and 0 port if the address or port isn't valid. */ + if (tor_addr_port_is_valid(or_addr, or_port, 0)) { + tor_addr_copy(&or_ap.addr, or_addr); + or_ap.port = or_port; + } else { + /* the family doesn't matter here, so make it IPv4 */ + tor_addr_make_null(&or_ap.addr, AF_INET); + or_ap.port = or_port = 0; + } + + if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) { + tor_addr_copy(&dir_ap.addr, dir_addr); + dir_ap.port = dir_port; + } else { + /* the family doesn't matter here, so make it IPv4 */ + tor_addr_make_null(&dir_ap.addr, AF_INET); + dir_ap.port = dir_port = 0; + } + + directory_initiate_command_rend(&or_ap, &dir_ap, digest, dir_purpose, router_purpose, indirection, resource, payload, payload_len, @@ -886,10 +1089,11 @@ is_sensitive_dir_purpose(uint8_t dir_purpose) } /** Same as directory_initiate_command(), but accepts rendezvous data to - * fetch a hidden service descriptor. */ + * fetch a hidden service descriptor, and takes its address & port arguments + * as tor_addr_port_t. */ static void -directory_initiate_command_rend(const tor_addr_t *_addr, - uint16_t or_port, uint16_t dir_port, +directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, + const tor_addr_port_t *dir_addr_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, dir_indirection_t indirection, @@ -898,19 +1102,33 @@ directory_initiate_command_rend(const tor_addr_t *_addr, time_t if_modified_since, const rend_data_t *rend_query) { + tor_assert(or_addr_port); + tor_assert(dir_addr_port); + tor_assert(or_addr_port->port || dir_addr_port->port); + tor_assert(digest); + dir_connection_t *conn; const or_options_t *options = get_options(); int socket_error = 0; - int use_begindir = directory_command_should_use_begindir(options, _addr, - or_port, router_purpose, indirection); + const char *begindir_reason = NULL; + /* Should the connection be to a relay's OR port (and inside that we will + * send our directory request)? */ + const int use_begindir = directory_command_should_use_begindir(options, + &or_addr_port->addr, or_addr_port->port, + router_purpose, indirection, + &begindir_reason); + /* Will the connection go via a three-hop Tor circuit? Note that this + * is separate from whether it will use_begindir. */ const int anonymized_connection = dirind_is_anon(indirection); - tor_addr_t addr; - tor_assert(_addr); - tor_assert(or_port || dir_port); - tor_assert(digest); - - tor_addr_copy(&addr, _addr); + /* What is the address we want to make the directory request to? If + * we're making a begindir request this is the ORPort of the relay + * we're contacting; if not a begindir request, this is its DirPort. + * Note that if anonymized_connection is true, we won't be initiating + * a connection directly to this address. */ + tor_addr_t addr; + tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr); + uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port; log_debug(LD_DIR, "anonymized %d, use_begindir %d.", anonymized_connection, use_begindir); @@ -924,6 +1142,14 @@ directory_initiate_command_rend(const tor_addr_t *_addr, (void)is_sensitive_dir_purpose; #endif + /* use encrypted begindir connections for everything except relays + * this provides better protection for directory fetches */ + if (!use_begindir && directory_must_use_begindir(options)) { + log_warn(LD_BUG, "Client could not use begindir connection: %s", + begindir_reason ? begindir_reason : "(NULL)"); + return; + } + /* ensure that we don't make direct connections when a SOCKS server is * configured. */ if (!anonymized_connection && !use_begindir && !options->HTTPProxy && @@ -933,11 +1159,25 @@ directory_initiate_command_rend(const tor_addr_t *_addr, return; } + /* Make sure that the destination addr and port we picked is viable. */ + if (!port || tor_addr_is_null(&addr)) { + static int logged_backtrace = 0; + log_warn(LD_DIR, + "Cannot make an outgoing %sconnection without %sPort.", + use_begindir ? "begindir " : "", + use_begindir ? "an OR" : "a Dir"); + if (!logged_backtrace) { + log_backtrace(LOG_INFO, LD_BUG, "Address came from"); + logged_backtrace = 1; + } + return; + } + conn = dir_connection_new(tor_addr_family(&addr)); /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); - conn->base_.port = use_begindir ? or_port : dir_port; + conn->base_.port = port; conn->base_.address = tor_dup_addr(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); @@ -960,16 +1200,13 @@ directory_initiate_command_rend(const tor_addr_t *_addr, if (options->HTTPProxy) { tor_addr_copy(&addr, &options->HTTPProxyAddr); - dir_port = options->HTTPProxyPort; + port = options->HTTPProxyPort; } switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, - dir_port, &socket_error)) { + port, &socket_error)) { case -1: - connection_dir_request_failed(conn); /* retry if we want */ - /* XXX we only pass 'conn' above, not 'resource', 'payload', - * etc. So in many situations it can't retry! -RD */ - connection_free(TO_CONN(conn)); + connection_mark_for_close(TO_CONN(conn)); return; case 1: /* start flushing conn */ @@ -984,8 +1221,12 @@ directory_initiate_command_rend(const tor_addr_t *_addr, /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } - } else { /* we want to connect via a tor connection */ + } else { + /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with + * begindir, maybe not with begindir) */ + entry_connection_t *linked_conn; + /* Anonymized tunneled connections can never share a circuit. * One-hop directory connections can share circuits with each other * but nothing else. */ @@ -1007,7 +1248,7 @@ directory_initiate_command_rend(const tor_addr_t *_addr, conn->base_.address, conn->base_.port, digest, SESSION_GROUP_DIRCONN, iso_flags, - use_begindir, conn->dirconn_direct); + use_begindir, !anonymized_connection); if (!linked_conn) { log_warn(LD_NET,"Making tunnel to dirserver failed."); connection_mark_for_close(TO_CONN(conn)); @@ -1113,6 +1354,23 @@ directory_get_consensus_url(const char *resource) return url; } +/** + * Copies the ipv6 from source to destination, subject to buffer size limit + * size. If decorate is true, makes sure the copied address is decorated. + */ +static void +copy_ipv6_address(char* destination, const char* source, size_t len, + int decorate) { + tor_assert(destination); + tor_assert(source); + + if (decorate && source[0] != '[') { + tor_snprintf(destination, len, "[%s]", source); + } else { + strlcpy(destination, source, len); + } +} + /** Queue an appropriate HTTP command on conn-\>outbuf. The other args * are as in directory_initiate_command(). */ @@ -1124,6 +1382,9 @@ directory_send_command(dir_connection_t *conn, { char proxystring[256]; char hoststring[128]; + /* NEEDS to be the same size hoststring. + Will be decorated with brackets around it if it is ipv6. */ + char decorated_address[128]; smartlist_t *headers = smartlist_new(); char *url; char request[8192]; @@ -1136,12 +1397,20 @@ directory_send_command(dir_connection_t *conn, if (resource) conn->requested_resource = tor_strdup(resource); + /* decorate the ip address if it is ipv6 */ + if (strchr(conn->base_.address, ':')) { + copy_ipv6_address(decorated_address, conn->base_.address, + sizeof(decorated_address), 1); + } else { + strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address)); + } + /* come up with a string for which Host: we want */ if (conn->base_.port == 80) { - strlcpy(hoststring, conn->base_.address, sizeof(hoststring)); + strlcpy(hoststring, decorated_address, sizeof(hoststring)); } else { - tor_snprintf(hoststring, sizeof(hoststring),"%s:%d", - conn->base_.address, conn->base_.port); + tor_snprintf(hoststring, sizeof(hoststring), "%s:%d", + decorated_address, conn->base_.port); } /* Format if-modified-since */ @@ -1255,7 +1524,8 @@ directory_send_command(dir_connection_t *conn, return; } - if (strlen(proxystring) + strlen(url) >= 4096) { + /* warn in the non-tunneled case */ + if (direct && (strlen(proxystring) + strlen(url) >= 4096)) { log_warn(LD_BUG, "Squid does not like URLs longer than 4095 bytes, and this " "one is %d bytes long: %s%s", @@ -1548,7 +1818,7 @@ load_downloaded_routers(const char *body, smartlist_t *which, added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, descriptor_digests, buf); - if (general) + if (added && general) control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, count_loading_descriptors_progress()); return added; @@ -1572,7 +1842,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) size_t body_len = 0, orig_len = 0; int status_code; time_t date_header = 0; - long delta; + long apparent_skew; compress_method_t compression; int plausible; int skewed = 0; @@ -1631,28 +1901,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * and the date header. (We used to check now-date_header, but that's * inaccurate if we spend a lot of time downloading.) */ - delta = conn->base_.timestamp_lastwritten - date_header; - if (labs(delta)>ALLOW_DIRECTORY_TIME_SKEW) { - char dbuf[64]; + apparent_skew = conn->base_.timestamp_lastwritten - date_header; + if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) { int trusted = router_digest_is_trusted_dir(conn->identity_digest); - format_time_interval(dbuf, sizeof(dbuf), delta); - log_fn(trusted ? LOG_WARN : LOG_INFO, - LD_HTTP, - "Received directory with skewed time (server '%s:%d'): " - "It seems that our clock is %s by %s, or that theirs is %s. " - "Tor requires an accurate clock to work: please check your time, " - "timezone, and date settings.", - conn->base_.address, conn->base_.port, - delta>0 ? "ahead" : "behind", dbuf, - delta>0 ? "behind" : "ahead"); + clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP, + "directory", "DIRSERV"); skewed = 1; /* don't check the recommended-versions line */ - if (trusted) - control_event_general_status(LOG_WARN, - "CLOCK_SKEW SKEW=%ld SOURCE=DIRSERV:%s:%d", - delta, conn->base_.address, conn->base_.port); } else { log_debug(LD_HTTP, "Time on received directory is within tolerance; " - "we are %ld seconds skewed. (That's okay.)", delta); + "we are %ld seconds skewed. (That's okay.)", apparent_skew); } } (void) skewed; /* skewed isn't used yet. */ @@ -1758,11 +2015,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) networkstatus_consensus_download_failed(0, flavname); return -1; } + + /* If we launched other fetches for this consensus, cancel them. */ + connection_dir_close_consensus_fetches(conn, flavname); + /* launches router downloads as needed */ routers_update_all_from_networkstatus(now, 3); update_microdescs_from_networkstatus(now); update_microdesc_downloads(now); - directory_info_has_arrived(now, 0); + directory_info_has_arrived(now, 0, 0); log_info(LD_DIR, "Successfully loaded consensus."); } @@ -1798,7 +2059,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { - directory_info_has_arrived(now, 0); + directory_info_has_arrived(now, 0, 0); log_info(LD_DIR, "Successfully loaded certificates from fetch."); } } else { @@ -1912,7 +2173,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (load_downloaded_routers(body, which, descriptor_digests, conn->router_purpose, conn->base_.address)) - directory_info_has_arrived(now, 0); + directory_info_has_arrived(now, 0, 0); } } if (which) { /* mark remaining ones as failed */ @@ -1963,8 +2224,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Mark remaining ones as failed. */ dir_microdesc_download_failed(which, status_code); } - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (mds && smartlist_len(mds)) { + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); + directory_info_has_arrived(now, 0, 1); + } SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); smartlist_free(which); smartlist_free(mds); @@ -2073,49 +2337,69 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { - #define SEND_HS_DESC_FAILED_EVENT() ( \ + #define SEND_HS_DESC_FAILED_EVENT(reason) ( \ control_event_hs_descriptor_failed(conn->rend_data, \ - conn->identity_digest) ) + conn->identity_digest, \ + reason) ) + #define SEND_HS_DESC_FAILED_CONTENT() ( \ + control_event_hs_descriptor_content(conn->rend_data->onion_address, \ + conn->requested_resource, \ + conn->identity_digest, \ + NULL) ) tor_assert(conn->rend_data); log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " "(%s))", (int)body_len, status_code, escaped(reason)); switch (status_code) { case 200: - switch (rend_cache_store_v2_desc_as_client(body, conn->rend_data)) { - case RCS_BADDESC: - case RCS_NOTDIR: /* Impossible */ - log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " - "Retrying at another directory."); - /* We'll retry when connection_about_to_close_connection() - * cleans this dir conn up. */ - SEND_HS_DESC_FAILED_EVENT(); - break; - case RCS_OKAY: - default: - /* success. notify pending connections about this. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor."); - control_event_hs_descriptor_received(conn->rend_data, - conn->identity_digest); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; - rend_client_desc_trynow(conn->rend_data->onion_address); - break; + { + rend_cache_entry_t *entry = NULL; + + if (rend_cache_store_v2_desc_as_client(body, + conn->requested_resource, conn->rend_data, &entry) < 0) { + log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " + "Retrying at another directory."); + /* We'll retry when connection_about_to_close_connection() + * cleans this dir conn up. */ + SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); + SEND_HS_DESC_FAILED_CONTENT(); + } else { + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + /* Should never be NULL here if we found the descriptor. */ + tor_assert(entry); + rend_get_service_id(entry->parsed->pk, service_id); + + /* success. notify pending connections about this. */ + log_info(LD_REND, "Successfully fetched v2 rendezvous " + "descriptor."); + control_event_hs_descriptor_received(service_id, + conn->rend_data, + conn->identity_digest); + control_event_hs_descriptor_content(service_id, + conn->requested_resource, + conn->identity_digest, + body); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; + rend_client_desc_trynow(service_id); + memwipe(service_id, 0, sizeof(service_id)); } break; + } case 404: /* Not there. We'll retry when * connection_about_to_close_connection() cleans this conn up. */ log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " "Retrying at another directory."); - SEND_HS_DESC_FAILED_EVENT(); + SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); + SEND_HS_DESC_FAILED_CONTENT(); break; case 400: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " "http status 400 (%s). Dirserver didn't like our " "v2 rendezvous query? Retrying at another directory.", escaped(reason)); - SEND_HS_DESC_FAILED_EVENT(); + SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); + SEND_HS_DESC_FAILED_CONTENT(); break; default: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " @@ -2124,31 +2408,45 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "Retrying at another directory.", status_code, escaped(reason), conn->base_.address, conn->base_.port); - SEND_HS_DESC_FAILED_EVENT(); + SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + SEND_HS_DESC_FAILED_CONTENT(); break; } } if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { + #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \ + control_event_hs_descriptor_upload_failed( \ + conn->identity_digest, \ + conn->rend_data->onion_address, \ + reason) ) log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " "(%s))", status_code, escaped(reason)); + /* Without the rend data, we'll have a problem identifying what has been + * uploaded for which service. */ + tor_assert(conn->rend_data); switch (status_code) { case 200: log_info(LD_REND, "Uploading rendezvous descriptor: finished with status " "200 (%s)", escaped(reason)); + control_event_hs_descriptor_uploaded(conn->identity_digest, + conn->rend_data->onion_address); + rend_service_desc_has_uploaded(conn->rend_data); break; case 400: log_warn(LD_REND,"http status 400 (%s) response from dirserver " "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", status_code, escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); break; } } @@ -2215,8 +2513,10 @@ connection_dir_process_inbuf(dir_connection_t *conn) MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE; if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) { - log_warn(LD_HTTP, "Too much data received from directory connection: " - "denial of service attempt, or you need to upgrade?"); + log_warn(LD_HTTP, + "Too much data received from directory connection (%s): " + "denial of service attempt, or you need to upgrade?", + conn->base_.address); connection_mark_for_close(TO_CONN(conn)); return -1; } @@ -2261,6 +2561,7 @@ write_http_status_line(dir_connection_t *conn, int status, log_warn(LD_BUG,"status line too long."); return; } + log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); connection_write_to_buf(buf, strlen(buf), TO_CONN(conn)); } @@ -2526,12 +2827,30 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) return (have >= need_at_least); } +/** Return the compression level we should use for sending a compressed + * response of size <b>n_bytes</b>. */ +STATIC zlib_compression_level_t +choose_compression_level(ssize_t n_bytes) +{ + if (! have_been_under_memory_pressure()) { + return HIGH_COMPRESSION; /* we have plenty of RAM. */ + } else if (n_bytes < 0) { + return HIGH_COMPRESSION; /* unknown; might be big. */ + } else if (n_bytes < 1024) { + return LOW_COMPRESSION; + } else if (n_bytes < 2048) { + return MEDIUM_COMPRESSION; + } else { + return HIGH_COMPRESSION; + } +} + /** Helper function: called when a dirserver gets a complete HTTP GET * request. Look for a request for a directory or for a rendezvous * service descriptor. On finding one, write a response into * conn-\>outbuf. If the request is unrecognized, send a 400. * Always return 0. */ -static int +STATIC int directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *req_body, size_t req_body_len) { @@ -2557,8 +2876,11 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if ((header = http_get_header(headers, "If-Modified-Since: "))) { struct tm tm; if (parse_http_time(header, &tm) == 0) { - if (tor_timegm(&tm, &if_modified_since)<0) + if (tor_timegm(&tm, &if_modified_since)<0) { if_modified_since = 0; + } else { + log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header)); + } } /* The correct behavior on a malformed If-Modified-Since header is to * act as if no If-Modified-Since header had been given. */ @@ -2684,10 +3006,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } if (1) { - struct in_addr in; tor_addr_t addr; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) { - tor_addr_from_ipv4h(&addr, ntohl(in.s_addr)); + if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, time(NULL)); @@ -2708,7 +3028,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_len(dir_fps) == 1 ? lifetime : 0); conn->fingerprint_stack = dir_fps; if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); /* Prime the connection with some data. */ conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS; @@ -2788,7 +3108,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, }); if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) { - write_http_status_line(conn, 503, "Directory busy, try again later."); + write_http_status_line(conn, 503, "Directory busy, try again later"); goto vote_done; } write_http_response_header(conn, body_len ? body_len : -1, compressed, @@ -2796,7 +3116,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_zlib(c, strlen(c), conn, 0)); connection_write_to_buf_zlib("", 0, conn, 1); @@ -2845,7 +3166,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, conn->fingerprint_stack = fps; if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(dlen)); connection_dirserv_flushed_some(conn); goto done; @@ -2913,7 +3235,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } write_http_response_header(conn, -1, compressed, cache_lifetime); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(dlen)); /* Prime the connection with some data. */ connection_dirserv_flushed_some(conn); } @@ -2982,13 +3305,14 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, len += c->cache_info.signed_descriptor_len); if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) { - write_http_status_line(conn, 503, "Directory busy, try again later."); + write_http_status_line(conn, 503, "Directory busy, try again later"); goto keys_done; } write_http_response_header(conn, compressed?-1:len, compressed, 60*60); if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, @@ -3005,13 +3329,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (options->HidServDirectoryV2 && - connection_dir_is_encrypted(conn) && + if (connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/")) { /* Handle v2 rendezvous descriptor fetch request. */ const char *descp; const char *query = url + strlen("/tor/rendezvous2/"); - if (strlen(query) == REND_DESC_ID_V2_LEN_BASE32) { + if (rend_valid_descriptor_id(query)) { log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'", safe_str(escaped(query))); switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { @@ -3143,6 +3466,13 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; + if (!public_server_mode(options)) { + log_info(LD_DIR, "Rejected dir post request from %s " + "since we're not a public relay.", conn->base_.address); + write_http_status_line(conn, 503, "Not acting as a public relay"); + goto done; + } + if (parse_http_url(headers, &url) < 0) { write_http_status_line(conn, 400, "Bad request"); return 0; @@ -3150,27 +3480,16 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); /* Handle v2 rendezvous service publish request. */ - if (options->HidServDirectoryV2 && - connection_dir_is_encrypted(conn) && + if (connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/publish")) { - switch (rend_cache_store_v2_desc_as_dir(body)) { - case RCS_NOTDIR: - log_info(LD_REND, "Rejected v2 rend descriptor (length %d) from %s " - "since we're not currently a hidden service directory.", - (int)body_len, conn->base_.address); - write_http_status_line(conn, 503, "Currently not acting as v2 " - "hidden service directory"); - break; - case RCS_BADDESC: - log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", - (int)body_len, conn->base_.address); - write_http_status_line(conn, 400, - "Invalid v2 service descriptor rejected"); - break; - case RCS_OKAY: - default: - write_http_status_line(conn, 200, "Service descriptor (v2) stored"); - log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); + if (rend_cache_store_v2_desc_as_dir(body) < 0) { + log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", + (int)body_len, conn->base_.address); + write_http_status_line(conn, 400, + "Invalid v2 service descriptor rejected"); + } else { + write_http_status_line(conn, 200, "Service descriptor (v2) stored"); + log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); } goto done; } @@ -3310,7 +3629,7 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn->base_.type == CONN_TYPE_DIR); /* Note that we have finished writing the directory response. For direct - * connections this means we're done, for tunneled connections its only + * connections this means we're done; for tunneled connections it's only * an intermediate step. */ if (conn->dirreq_id) geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, @@ -3351,8 +3670,38 @@ connection_dir_finished_flushing(dir_connection_t *conn) return 0; } +/* We just got a new consensus! If there are other in-progress requests + * for this consensus flavor (for example because we launched several in + * parallel), cancel them. + * + * We do this check here (not just in + * connection_ap_handshake_attach_circuit()) to handle the edge case where + * a consensus fetch begins and ends before some other one tries to attach to + * a circuit, in which case the other one won't know that we're all happy now. + * + * Don't mark the conn that just gave us the consensus -- otherwise we + * would end up double-marking it when it cleans itself up. + */ +static void +connection_dir_close_consensus_fetches(dir_connection_t *except_this_one, + const char *resource) +{ + smartlist_t *conns_to_close = + connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS, + resource); + SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) { + if (d == except_this_one) + continue; + log_info(LD_DIR, "Closing consensus fetch (to %s) since one " + "has just arrived.", TO_CONN(d)->address); + connection_mark_for_close(TO_CONN(d)); + } SMARTLIST_FOREACH_END(d); + smartlist_free(conns_to_close); +} + /** Connected handler for directory connections: begin sending data to the - * server */ + * server, and return 0. + * Only used when connections don't immediately connect. */ int connection_dir_finished_connecting(dir_connection_t *conn) { @@ -3363,84 +3712,229 @@ connection_dir_finished_connecting(dir_connection_t *conn) log_debug(LD_HTTP,"Dir connection to router %s:%u established.", conn->base_.address,conn->base_.port); - conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; /* start flushing conn */ + /* start flushing conn */ + conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; return 0; } /** Decide which download schedule we want to use based on descriptor type - * in <b>dls</b> and whether we are acting as directory <b>server</b>, and - * then return a list of int pointers defining download delays in seconds. - * Helper function for download_status_increment_failure() and - * download_status_reset(). */ -static const smartlist_t * -find_dl_schedule_and_len(download_status_t *dls, int server) + * in <b>dls</b> and <b>options</b>. + * Then return a list of int pointers defining download delays in seconds. + * Helper function for download_status_increment_failure(), + * download_status_reset(), and download_status_increment_attempt(). */ +STATIC const smartlist_t * +find_dl_schedule(download_status_t *dls, const or_options_t *options) { + const int dir_server = dir_server_mode(options); + const int multi_d = networkstatus_consensus_can_use_multiple_directories( + options); + const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping( + time(NULL)); + const int use_fallbacks = networkstatus_consensus_can_use_extra_fallbacks( + options); switch (dls->schedule) { case DL_SCHED_GENERIC: - if (server) - return get_options()->TestingServerDownloadSchedule; - else - return get_options()->TestingClientDownloadSchedule; + if (dir_server) { + return options->TestingServerDownloadSchedule; + } else { + return options->TestingClientDownloadSchedule; + } case DL_SCHED_CONSENSUS: - if (server) - return get_options()->TestingServerConsensusDownloadSchedule; - else - return get_options()->TestingClientConsensusDownloadSchedule; + if (!multi_d) { + return options->TestingServerConsensusDownloadSchedule; + } else { + if (we_are_bootstrapping) { + if (!use_fallbacks) { + /* A bootstrapping client without extra fallback directories */ + return + options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule; + } else if (dls->want_authority) { + /* A bootstrapping client with extra fallback directories, but + * connecting to an authority */ + return + options->ClientBootstrapConsensusAuthorityDownloadSchedule; + } else { + /* A bootstrapping client connecting to extra fallback directories + */ + return + options->ClientBootstrapConsensusFallbackDownloadSchedule; + } + } else { + return options->TestingClientConsensusDownloadSchedule; + } + } case DL_SCHED_BRIDGE: - return get_options()->TestingBridgeDownloadSchedule; + return options->TestingBridgeDownloadSchedule; default: tor_assert(0); } + + /* Impossible, but gcc will fail with -Werror without a `return`. */ + return NULL; } -/** Called when an attempt to download <b>dls</b> has failed with HTTP status +/* Find the current delay for dls based on schedule. + * Set dls->next_attempt_at based on now, and return the delay. + * Helper for download_status_increment_failure and + * download_status_increment_attempt. */ +STATIC int +download_status_schedule_get_delay(download_status_t *dls, + const smartlist_t *schedule, + time_t now) +{ + tor_assert(dls); + tor_assert(schedule); + + int delay = INT_MAX; + uint8_t dls_schedule_position = (dls->increment_on + == DL_SCHED_INCREMENT_ATTEMPT + ? dls->n_download_attempts + : dls->n_download_failures); + + if (dls_schedule_position < smartlist_len(schedule)) + delay = *(int *)smartlist_get(schedule, dls_schedule_position); + else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD) + delay = INT_MAX; + else + delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); + + /* A negative delay makes no sense. Knowing that delay is + * non-negative allows us to safely do the wrapping check below. */ + tor_assert(delay >= 0); + + /* Avoid now+delay overflowing INT_MAX, by comparing with a subtraction + * that won't overflow (since delay is non-negative). */ + if (delay < INT_MAX && now <= INT_MAX - delay) { + dls->next_attempt_at = now+delay; + } else { + dls->next_attempt_at = TIME_MAX; + } + + return delay; +} + +/* Log a debug message about item, which increments on increment_action, has + * incremented dls_n_download_increments times. The message varies based on + * was_schedule_incremented (if not, not_incremented_response is logged), and + * the values of increment, dls_next_attempt_at, and now. + * Helper for download_status_increment_failure and + * download_status_increment_attempt. */ +static void +download_status_log_helper(const char *item, int was_schedule_incremented, + const char *increment_action, + const char *not_incremented_response, + uint8_t dls_n_download_increments, int increment, + time_t dls_next_attempt_at, time_t now) +{ + if (item) { + if (!was_schedule_incremented) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.", + item, increment_action, (int)dls_n_download_increments, + not_incremented_response); + else if (increment == 0) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.", + item, increment_action, (int)dls_n_download_increments); + else if (dls_next_attempt_at < TIME_MAX) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.", + item, increment_action, (int)dls_n_download_increments, + (int)(dls_next_attempt_at-now)); + else + log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.", + item, increment_action, (int)dls_n_download_increments); + } +} + +/** Determine when a failed download attempt should be retried. + * Called when an attempt to download <b>dls</b> has failed with HTTP status * <b>status_code</b>. Increment the failure count (if the code indicates a - * real failure) and set <b>dls</b>-\>next_attempt_at to an appropriate time - * in the future. */ + * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to + * an appropriate time in the future and return it. + * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the + * failure count, and return a time in the far future for the next attempt (to + * avoid an immediate retry). */ time_t download_status_increment_failure(download_status_t *dls, int status_code, const char *item, int server, time_t now) { - const smartlist_t *schedule; - int increment; + int increment = -1; tor_assert(dls); + + /* only count the failure if it's permanent, or we're a server */ if (status_code != 503 || server) { if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) ++dls->n_download_failures; } - schedule = find_dl_schedule_and_len(dls, server); + if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { + /* We don't find out that a failure-based schedule has attempted a + * connection until that connection fails. + * We'll never find out about successful connections, but this doesn't + * matter, because schedules are reset after a successful download. + */ + if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) + ++dls->n_download_attempts; - if (dls->n_download_failures < smartlist_len(schedule)) - increment = *(int *)smartlist_get(schedule, dls->n_download_failures); - else if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD) - increment = INT_MAX; - else - increment = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); + /* only return a failure retry time if this schedule increments on failures + */ + const smartlist_t *schedule = find_dl_schedule(dls, get_options()); + increment = download_status_schedule_get_delay(dls, schedule, now); + } - if (increment < INT_MAX) - dls->next_attempt_at = now+increment; - else - dls->next_attempt_at = TIME_MAX; + download_status_log_helper(item, !dls->increment_on, "failed", + "concurrently", dls->n_download_failures, + increment, dls->next_attempt_at, now); - if (item) { - if (increment == 0) - log_debug(LD_DIR, "%s failed %d time(s); I'll try again immediately.", - item, (int)dls->n_download_failures); - else if (dls->next_attempt_at < TIME_MAX) - log_debug(LD_DIR, "%s failed %d time(s); I'll try again in %d seconds.", - item, (int)dls->n_download_failures, - (int)(dls->next_attempt_at-now)); - else - log_debug(LD_DIR, "%s failed %d time(s); Giving up for a while.", - item, (int)dls->n_download_failures); + if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) { + /* stop this schedule retrying on failure, it will launch concurrent + * connections instead */ + return TIME_MAX; + } else { + return dls->next_attempt_at; + } +} + +/** Determine when the next download attempt should be made when using an + * attempt-based (potentially concurrent) download schedule. + * Called when an attempt to download <b>dls</b> is being initiated. + * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an + * appropriate time in the future and return it. + * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment + * the attempts, and return a time in the far future (to avoid launching a + * concurrent attempt). */ +time_t +download_status_increment_attempt(download_status_t *dls, const char *item, + time_t now) +{ + int delay = -1; + tor_assert(dls); + + if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { + /* this schedule should retry on failure, and not launch any concurrent + attempts */ + log_info(LD_BUG, "Tried to launch an attempt-based connection on a " + "failure-based schedule."); + return TIME_MAX; } + + if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) + ++dls->n_download_attempts; + + const smartlist_t *schedule = find_dl_schedule(dls, get_options()); + delay = download_status_schedule_get_delay(dls, schedule, now); + + download_status_log_helper(item, dls->increment_on, "attempted", + "on failure", dls->n_download_attempts, + delay, dls->next_attempt_at, now); + return dls->next_attempt_at; } /** Reset <b>dls</b> so that it will be considered downloadable * immediately, and/or to show that we don't need it anymore. * + * Must be called to initialise a download schedule, otherwise the zeroth item + * in the schedule will never be used. + * * (We find the zeroth element of the download schedule, and set * next_attempt_at to be the appropriate offset from 'now'. In most * cases this means setting it to 'now', so the item will be immediately @@ -3449,11 +3943,16 @@ download_status_increment_failure(download_status_t *dls, int status_code, void download_status_reset(download_status_t *dls) { - const smartlist_t *schedule = find_dl_schedule_and_len( - dls, get_options()->DirPort_set); + if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD + || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD) + return; /* Don't reset this. */ + + const smartlist_t *schedule = find_dl_schedule(dls, get_options()); dls->n_download_failures = 0; + dls->n_download_attempts = 0; dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0); + /* Don't reset dls->want_authority or dls->increment_on */ } /** Return the number of failures on <b>dls</b> since the last success (if @@ -3464,6 +3963,22 @@ download_status_get_n_failures(const download_status_t *dls) return dls->n_download_failures; } +/** Return the number of attempts to download <b>dls</b> since the last success + * (if any). This can differ from download_status_get_n_failures() due to + * outstanding concurrent attempts. */ +int +download_status_get_n_attempts(const download_status_t *dls) +{ + return dls->n_download_attempts; +} + +/** Return the next time to attempt to download <b>dls</b>. */ +time_t +download_status_get_next_attempt_at(const download_status_t *dls) +{ + return dls->next_attempt_at; +} + /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, * either as descriptor digests or as identity digests based on |