diff options
Diffstat (limited to 'src/or/dirserv.c')
| -rw-r--r-- | src/or/dirserv.c | 1714 |
1 files changed, 695 insertions, 1019 deletions
diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 177009208d..ae1975f166 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@ -13,10 +13,11 @@ #include "command.h" #include "connection.h" #include "connection_or.h" +#include "conscache.h" +#include "consdiffmgr.h" #include "control.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "hibernate.h" #include "keypin.h" #include "main.h" @@ -31,6 +32,9 @@ #include "routerparse.h" #include "routerset.h" #include "torcert.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" /** * \file dirserv.c @@ -41,7 +45,7 @@ * the directory authority functionality. The directory.c module delegates * here in order to handle incoming requests from clients, via * connection_dirserv_flushed_some() and its kin. In order to save RAM, this - * module is reponsible for spooling directory objects (in whole or in part) + * module is responsible for spooling directory objects (in whole or in part) * onto buf_t instances, and then closing the dir_connection_t once the * objects are totally flushed. * @@ -80,14 +84,22 @@ dirserv_get_status_impl(const char *fp, const char *nickname, int severity); static void clear_cached_dir(cached_dir_t *d); static const signed_descriptor_t *get_signed_descriptor_by_fp( - const char *fp, - int extrainfo, - time_t publish_cutoff); + const uint8_t *fp, + int extrainfo); static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); -static uint32_t dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri); static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri); +static int spooled_resource_lookup_body(const spooled_resource_t *spooled, + int conn_is_encrypted, + const uint8_t **body_out, + size_t *size_out, + time_t *published_out); +static cached_dir_t *spooled_resource_lookup_cached_dir( + const spooled_resource_t *spooled, + time_t *published_out); +static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp); + /************** Fingerprint handling code ************/ /* 1 Historically used to indicate Named */ @@ -247,11 +259,12 @@ dirserv_load_fingerprint_file(void) * identity to stop doing so. This is going to be essential for good identity * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could * just sign fake descriptors missing the Ed25519 key. But we won't actually - * be able to prevent that kind of thing until we're confident that there - * isn't actually a legit reason to downgrade to 0.2.5. So for now, we have - * to leave this #undef. + * be able to prevent that kind of thing until we're confident that there isn't + * actually a legit reason to downgrade to 0.2.5. Now we are not recommending + * 0.2.5 anymore so there is no reason to keep the #undef. */ -#undef DISABLE_DISABLING_ED25519 + +#define DISABLE_DISABLING_ED25519 /** Check whether <b>router</b> has a nickname/identity key combination that * we recognize from the fingerprint list, or an IP we automatically act on @@ -273,6 +286,13 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg, return FP_REJECT; } + /* Check for the more usual versions to reject a router first. */ + const uint32_t r = dirserv_get_status_impl(d, router->nickname, + router->addr, router->or_port, + router->platform, msg, severity); + if (r) + return r; + /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc, * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha. * But just in case a relay doesn't provide or lies about its version, or @@ -319,13 +339,11 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg, } return FP_REJECT; } -#endif +#endif /* defined(DISABLE_DISABLING_ED25519) */ } } - return dirserv_get_status_impl(d, router->nickname, - router->addr, router->or_port, - router->platform, msg, severity); + return 0; } /** Return true if there is no point in downloading the router described by @@ -577,6 +595,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, !general ? router_purpose_to_string(purpose) : "", !general ? "\n" : "")<0) { *msg = "Couldn't format annotations"; + /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is + * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */ return -1; } @@ -660,9 +680,6 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) ri->nickname, source, (int)ri->cache_info.signed_descriptor_len, MAX_DESCRIPTOR_UPLOAD_SIZE); *msg = "Router descriptor was too large."; - control_event_or_authdir_new_descriptor("REJECTED", - ri->cache_info.signed_descriptor_body, - desclen, *msg); r = ROUTER_AUTHDIR_REJECTS; goto fail; } @@ -681,9 +698,6 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) router_describe(ri), source); *msg = "Not replacing router descriptor; no information has changed since " "the last one with this identity."; - control_event_or_authdir_new_descriptor("DROPPED", - ri->cache_info.signed_descriptor_body, - desclen, *msg); r = ROUTER_IS_ALREADY_KNOWN; goto fail; } @@ -691,10 +705,19 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) /* Do keypinning again ... this time, to add the pin if appropriate */ int keypin_status; if (ri->cache_info.signing_key_cert) { + ed25519_public_key_t *pkey = &ri->cache_info.signing_key_cert->signing_key; + /* First let's validate this pubkey before pinning it */ + if (ed25519_validate_pubkey(pkey) < 0) { + log_warn(LD_DIRSERV, "Received bad key from %s (source %s)", + router_describe(ri), source); + routerinfo_free(ri); + return ROUTER_AUTHDIR_REJECTS; + } + + /* Now pin it! */ keypin_status = keypin_check_and_add( (const uint8_t*)ri->cache_info.identity_digest, - ri->cache_info.signing_key_cert->signing_key.pubkey, - ! key_pinning); + pkey->pubkey, ! key_pinning); } else { keypin_status = keypin_check_lone_rsa( (const uint8_t*)ri->cache_info.identity_digest); @@ -707,7 +730,12 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because " "its key did not match an older RSA/Ed25519 keypair", router_describe(ri), source); - *msg = "Looks like your keypair does not match its older value."; + *msg = "Looks like your keypair has changed? This authority previously " + "recorded a different RSA identity for this Ed25519 identity (or vice " + "versa.) Did you replace or copy some of your key files, but not " + "the others? You should either restore the expected keypair, or " + "delete your keys and restart Tor to start your relay with a new " + "identity."; r = ROUTER_AUTHDIR_REJECTS; goto fail; } @@ -724,14 +752,11 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) r = router_add_to_routerlist(ri, msg, 0, 0); if (!WRA_WAS_ADDED(r)) { /* unless the routerinfo was fine, just out-of-date */ - if (WRA_WAS_REJECTED(r)) - control_event_or_authdir_new_descriptor("REJECTED", desc, desclen, *msg); log_info(LD_DIRSERV, "Did not add descriptor from '%s' (source: %s): %s.", nickname, source, *msg ? *msg : "(no message)"); } else { smartlist_t *changed; - control_event_or_authdir_new_descriptor("ACCEPTED", desc, desclen, *msg); changed = smartlist_new(); smartlist_add(changed, ri); @@ -834,13 +859,13 @@ directory_remove_invalid(void) SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) { const char *msg = NULL; + const char *description; routerinfo_t *ent = node->ri; - char description[NODE_DESC_BUF_LEN]; uint32_t r; if (!ent) continue; r = dirserv_router_get_status(ent, &msg, LOG_INFO); - router_get_description(description, ent); + description = router_describe(ent); if (r & FP_REJECT) { log_info(LD_DIRSERV, "Router %s is now rejected: %s", description, msg?msg:""); @@ -867,6 +892,9 @@ directory_remove_invalid(void) * Allocate and return a description of the status of the server <b>desc</b>, * for use in a v1-style router-status line. The server is listed * as running iff <b>is_live</b> is true. + * + * This is deprecated: it's only used for controllers that want outputs in + * the old format. */ static char * list_single_server_status(const routerinfo_t *desc, int is_live) @@ -894,7 +922,7 @@ list_single_server_status(const routerinfo_t *desc, int is_live) } /* DOCDOC running_long_enough_to_decide_unreachable */ -static inline int +int running_long_enough_to_decide_unreachable(void) { return time_of_process_start @@ -927,7 +955,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) tor_assert(node); if (router_is_me(router)) { - /* We always know if we are down ourselves. */ + /* We always know if we are shutting down or hibernating ourselves. */ answer = ! we_are_hibernating(); } else if (router->is_hibernating && (router->cache_info.published_on + @@ -979,6 +1007,9 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. + * + * This is deprecated: it's only used for controllers that want outputs in + * the old format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -1011,7 +1042,7 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, if (!node->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); - smartlist_add(rs_entries, tor_strdup(name_buf)); + smartlist_add_strdup(rs_entries, name_buf); } else if (ri->cache_info.published_on >= cutoff) { smartlist_add(rs_entries, list_single_server_status(ri, node->is_running)); @@ -1026,59 +1057,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, return 0; } -/** Given a (possibly empty) list of config_line_t, each line of which contains - * a list of comma-separated version numbers surrounded by optional space, - * allocate and return a new string containing the version numbers, in order, - * separated by commas. Used to generate Recommended(Client|Server)?Versions - */ -char * -format_recommended_version_list(const config_line_t *ln, int warn) -{ - smartlist_t *versions; - char *result; - versions = smartlist_new(); - for ( ; ln; ln = ln->next) { - smartlist_split_string(versions, ln->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - } - - /* Handle the case where a dirauth operator has accidentally made some - * versions space-separated instead of comma-separated. */ - smartlist_t *more_versions = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(versions, char *, v) { - if (strchr(v, ' ')) { - if (warn) - log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. " - "(These are supposed to be comma-separated; I'll pretend you " - "used commas instead.)", escaped(v)); - SMARTLIST_DEL_CURRENT(versions, v); - smartlist_split_string(more_versions, v, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - tor_free(v); - } - } SMARTLIST_FOREACH_END(v); - smartlist_add_all(versions, more_versions); - smartlist_free(more_versions); - - /* Check to make sure everything looks like a version. */ - if (warn) { - SMARTLIST_FOREACH_BEGIN(versions, const char *, v) { - tor_version_t ver; - if (tor_version_parse(v, &ver) < 0) { - log_warn(LD_DIRSERV, "Recommended version %s does not look valid. " - " (I'll include it anyway, since you told me to.)", - escaped(v)); - } - } SMARTLIST_FOREACH_END(v); - } - - sort_version_list(versions, 1); - result = smartlist_join_strings(versions,",",0,NULL); - SMARTLIST_FOREACH(versions,char *,s,tor_free(s)); - smartlist_free(versions); - return result; -} - /** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid, * not hibernating, having observed bw greater 0, and not too old. Else * return 0. @@ -1093,7 +1071,7 @@ router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) if (!node->is_running || !node->is_valid || ri->is_hibernating) { return 0; } - /* Only require bandwith capacity in non-test networks, or + /* Only require bandwidth capacity in non-test networks, or * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */ if (!ri->bandwidthcapacity) { if (get_options()->TestingTorNetwork) { @@ -1163,8 +1141,10 @@ directory_fetches_dir_info_later(const or_options_t *options) return options->UseBridges != 0; } -/** Return true iff we want to fetch and keep certificates for authorities +/** Return true iff we want to serve certificates for authorities * that we don't acknowledge as authorities ourself. + * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch + * and keep these certificates. */ int directory_caches_unknown_auth_certs(const or_options_t *options) @@ -1172,11 +1152,14 @@ directory_caches_unknown_auth_certs(const or_options_t *options) return dir_server_mode(options) || options->BridgeRelay; } -/** Return 1 if we want to keep descriptors, networkstatuses, etc around. +/** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc * Else return 0. * Check options->DirPort_set and directory_permits_begindir_requests() * to see if we are willing to serve these directory documents to others via * the DirPort and begindir-over-ORPort, respectively. + * + * To check if we should fetch documents, use we_want_to_fetch_flavor and + * we_want_to_fetch_unknown_auth_certs instead of this function. */ int directory_caches_dir_info(const or_options_t *options) @@ -1191,7 +1174,7 @@ directory_caches_dir_info(const or_options_t *options) should_refuse_unknown_exits(options); } -/** Return 1 if we want to allow remote people to ask us directory +/** Return 1 if we want to allow remote clients to ask us directory * requests via the "begin_dir" interface, which doesn't require * having any separate port open. */ int @@ -1240,8 +1223,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len), + d->dir, d->dir_len, ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; @@ -1252,7 +1235,7 @@ static void clear_cached_dir(cached_dir_t *d) { tor_free(d->dir); - tor_free(d->dir_z); + tor_free(d->dir_compressed); memset(d, 0, sizeof(cached_dir_t)); } @@ -1275,6 +1258,7 @@ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, const char *flavor_name, const common_digests_t *digests, + const uint8_t *sha3_as_signed, time_t published) { cached_dir_t *new_networkstatus; @@ -1284,6 +1268,8 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); + memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, + DIGEST256_LEN); old_networkstatus = strmap_set(cached_consensuses, flavor_name, new_networkstatus); if (old_networkstatus) @@ -1416,7 +1402,7 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router, * tests aren't instant. If we haven't been running long enough, * trust the relay. */ - if (stats_n_seconds_working > + if (get_uptime() > get_options()->MinUptimeHidServDirectoryV2 * 1.1) uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now), real_uptime(router, now)); @@ -1460,6 +1446,24 @@ router_counts_toward_thresholds(const node_t *node, time_t now, (have_mbw || !require_mbw); } +/** Look through the routerlist, and using the measured bandwidth cache count + * how many measured bandwidths we know. This is used to decide whether we + * ever trust advertised bandwidths for purposes of assigning flags. */ +void +dirserv_count_measured_bws(const smartlist_t *routers) +{ + /* Initialize this first */ + routers_with_measured_bw = 0; + + /* Iterate over the routerlist and count measured bandwidths */ + SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { + /* Check if we know a measured bandwidth for this one */ + if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) { + ++routers_with_measured_bw; + } + } SMARTLIST_FOREACH_END(ri); +} + /** Look through the routerlist, the Mean Time Between Failure history, and * the Weighted Fractional Uptime history, and use them to set thresholds for * the Stable, Fast, and Guard flags. Update the fields stable_uptime, @@ -1467,7 +1471,7 @@ router_counts_toward_thresholds(const node_t *node, time_t now, * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits. * * Also, set the is_exit flag of each router appropriately. */ -static void +void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil) { int n_active, n_active_nonexit, n_familiar; @@ -1518,15 +1522,21 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil) node->ri && node->ri->purpose != ROUTER_PURPOSE_BRIDGE) continue; + + routerinfo_t *ri = node->ri; + if (ri) { + node->is_exit = (!router_exit_policy_rejects_all(ri) && + exit_policy_is_general_exit(ri->exit_policy)); + } + if (router_counts_toward_thresholds(node, now, omit_as_sybil, require_mbw)) { - routerinfo_t *ri = node->ri; const char *id = node->identity; uint32_t bw_kb; + /* resolve spurious clang shallow analysis null pointer errors */ tor_assert(ri); - node->is_exit = (!router_exit_policy_rejects_all(ri) && - exit_policy_is_general_exit(ri->exit_policy)); + uptimes[n_active] = (uint32_t)real_uptime(ri, now); mtbfs[n_active] = rep_hist_get_stability(id, now); tks [n_active] = rep_hist_get_weighted_time_known(id, now); @@ -1692,7 +1702,7 @@ dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, } /** Clear and free the measured bandwidth cache */ -STATIC void +void dirserv_clear_measured_bw_cache(void) { if (mbw_cache) { @@ -1724,18 +1734,10 @@ dirserv_expire_measured_bw_cache(time_t now) } } -/** Get the current size of the measured bandwidth cache */ -STATIC int -dirserv_get_measured_bw_cache_size(void) -{ - if (mbw_cache) return digestmap_size(mbw_cache); - else return 0; -} - /** Query the cache by identity digest, return value indicates whether * we found it. The bw_out and as_of_out pointers receive the cached * bandwidth value and the time it was cached if not NULL. */ -STATIC int +int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, time_t *as_of_out) { @@ -1756,61 +1758,18 @@ dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, } /** Predicate wrapper for dirserv_query_measured_bw_cache() */ -STATIC int +int dirserv_has_measured_bw(const char *node_id) { return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL); } -/** Get the best estimate of a router's bandwidth for dirauth purposes, - * preferring measured to advertised values if available. */ - -static uint32_t -dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri) -{ - uint32_t bw_kb = 0; - /* - * Yeah, measured bandwidths in measured_bw_line_t are (implicitly - * signed) longs and the ones router_get_advertised_bandwidth() returns - * are uint32_t. - */ - long mbw_kb = 0; - - if (ri) { - /* - * * First try to see if we have a measured bandwidth; don't bother with - * as_of_out here, on the theory that a stale measured bandwidth is still - * better to trust than an advertised one. - */ - if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest, - &mbw_kb, NULL)) { - /* Got one! */ - bw_kb = (uint32_t)mbw_kb; - } else { - /* If not, fall back to advertised */ - bw_kb = router_get_advertised_bandwidth(ri) / 1000; - } - } - - return bw_kb; -} - -/** Look through the routerlist, and using the measured bandwidth cache count - * how many measured bandwidths we know. This is used to decide whether we - * ever trust advertised bandwidths for purposes of assigning flags. */ -static void -dirserv_count_measured_bws(const smartlist_t *routers) +/** Get the current size of the measured bandwidth cache */ +int +dirserv_get_measured_bw_cache_size(void) { - /* Initialize this first */ - routers_with_measured_bw = 0; - - /* Iterate over the routerlist and count measured bandwidths */ - SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { - /* Check if we know a measured bandwidth for this one */ - if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) { - ++routers_with_measured_bw; - } - } SMARTLIST_FOREACH_END(ri); + if (mbw_cache) return digestmap_size(mbw_cache); + else return 0; } /** Return the bandwidth we believe for assigning flags; prefer measured @@ -1873,44 +1832,31 @@ dirserv_get_flag_thresholds_line(void) return result; } -/** Given a platform string as in a routerinfo_t (possibly null), return a - * newly allocated version string for a networkstatus document, or NULL if the - * platform doesn't give a Tor version. */ -static char * -version_from_platform(const char *platform) -{ - if (platform && !strcmpstart(platform, "Tor ")) { - const char *eos = find_whitespace(platform+4); - if (eos && !strcmpstart(eos, " (r")) { - /* XXXX Unify this logic with the other version extraction - * logic in routerparse.c. */ - eos = find_whitespace(eos+1); - } - if (eos) { - return tor_strndup(platform, eos-platform); - } - } - return NULL; -} - /** Helper: write the router-status information in <b>rs</b> into a newly * allocated character buffer. Use the same format as in network-status * documents. If <b>version</b> is non-NULL, add a "v" line for the platform. + * + * consensus_method is the current consensus method when format is + * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other + * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD. + * * Return 0 on success, -1 on failure. * * The format argument has one of the following values: * NS_V2 - Output an entry suitable for a V2 NS opinion document * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * for consensus_method. * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc - * consensus entry. + * consensus entry for consensus_method. * NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present, * it contains additional information for the vote. - * NS_CONTROL_PORT - Output a NS document for the control port + * NS_CONTROL_PORT - Output a NS document for the control port. */ char * routerstatus_format_entry(const routerstatus_t *rs, const char *version, const char *protocols, routerstatus_format_type_t format, + int consensus_method, const vote_routerstatus_t *vrs) { char *summary; @@ -1941,8 +1887,10 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, * networkstatus_type_t values, with an additional control port value * added -MP */ - /* V3 microdesc consensuses don't have "a" lines. */ - if (format == NS_V3_CONSENSUS_MICRODESC) + /* V3 microdesc consensuses only have "a" lines in later consensus methods + */ + if (format == NS_V3_CONSENSUS_MICRODESC && + consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS) goto done; /* Possible "a" line. At most one for now. */ @@ -1951,7 +1899,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport)); } - if (format == NS_V3_CONSENSUS) + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) goto done; smartlist_add_asprintf(chunks, @@ -2043,7 +1991,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, vrs->status.guardfraction_percentage); } - smartlist_add(chunks, tor_strdup("\n")); + smartlist_add_strdup(chunks, "\n"); if (desc) { summary = policy_summarize(desc->exit_policy, AF_INET); @@ -2053,7 +2001,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, if (format == NS_V3_VOTE && vrs) { if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) { - smartlist_add(chunks, tor_strdup("id ed25519 none\n")); + smartlist_add_strdup(chunks, "id ed25519 none\n"); } else { char ed_b64[BASE64_DIGEST256_LEN+1]; digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id); @@ -2072,153 +2020,9 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, return result; } -/** Helper for sorting: compares two routerinfos first by address, and then by - * descending order of "usefulness". (An authority is more useful than a - * non-authority; a running router is more useful than a non-running router; - * and a router with more bandwidth is more useful than one with less.) - **/ -static int -compare_routerinfo_by_ip_and_bw_(const void **a, const void **b) -{ - routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; - int first_is_auth, second_is_auth; - uint32_t bw_kb_first, bw_kb_second; - const node_t *node_first, *node_second; - int first_is_running, second_is_running; - - /* we return -1 if first should appear before second... that is, - * if first is a better router. */ - if (first->addr < second->addr) - return -1; - else if (first->addr > second->addr) - return 1; - - /* Potentially, this next bit could cause k n lg n memeq calls. But in - * reality, we will almost never get here, since addresses will usually be - * different. */ - - first_is_auth = - router_digest_is_trusted_dir(first->cache_info.identity_digest); - second_is_auth = - router_digest_is_trusted_dir(second->cache_info.identity_digest); - - if (first_is_auth && !second_is_auth) - return -1; - else if (!first_is_auth && second_is_auth) - return 1; - - node_first = node_get_by_id(first->cache_info.identity_digest); - node_second = node_get_by_id(second->cache_info.identity_digest); - first_is_running = node_first && node_first->is_running; - second_is_running = node_second && node_second->is_running; - - if (first_is_running && !second_is_running) - return -1; - else if (!first_is_running && second_is_running) - return 1; - - bw_kb_first = dirserv_get_bandwidth_for_router_kb(first); - bw_kb_second = dirserv_get_bandwidth_for_router_kb(second); - - if (bw_kb_first > bw_kb_second) - return -1; - else if (bw_kb_first < bw_kb_second) - return 1; - - /* They're equal! Compare by identity digest, so there's a - * deterministic order and we avoid flapping. */ - return fast_memcmp(first->cache_info.identity_digest, - second->cache_info.identity_digest, - DIGEST_LEN); -} - -/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t - * whose keys are the identity digests of those routers that we're going to - * exclude for Sybil-like appearance. */ -static digestmap_t * -get_possible_sybil_list(const smartlist_t *routers) -{ - const or_options_t *options = get_options(); - digestmap_t *omit_as_sybil; - smartlist_t *routers_by_ip = smartlist_new(); - uint32_t last_addr; - int addr_count; - /* Allow at most this number of Tor servers on a single IP address, ... */ - int max_with_same_addr = options->AuthDirMaxServersPerAddr; - /* ... unless it's a directory authority, in which case allow more. */ - int max_with_same_addr_on_authority = options->AuthDirMaxServersPerAuthAddr; - if (max_with_same_addr <= 0) - max_with_same_addr = INT_MAX; - if (max_with_same_addr_on_authority <= 0) - max_with_same_addr_on_authority = INT_MAX; - - smartlist_add_all(routers_by_ip, routers); - smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_); - omit_as_sybil = digestmap_new(); - - last_addr = 0; - addr_count = 0; - SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) { - if (last_addr != ri->addr) { - last_addr = ri->addr; - addr_count = 1; - } else if (++addr_count > max_with_same_addr) { - if (!router_addr_is_trusted_dir(ri->addr) || - addr_count > max_with_same_addr_on_authority) - digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri); - } - } SMARTLIST_FOREACH_END(ri); - - smartlist_free(routers_by_ip); - return omit_as_sybil; -} - -/** If there are entries in <b>routers</b> with exactly the same ed25519 keys, - * remove the older one. If they are exactly the same age, remove the one - * with the greater descriptor digest. May alter the order of the list. */ -static void -routers_make_ed_keys_unique(smartlist_t *routers) -{ - routerinfo_t *ri2; - digest256map_t *by_ed_key = digest256map_new(); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - ri->omit_from_vote = 0; - if (ri->cache_info.signing_key_cert == NULL) - continue; /* No ed key */ - const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey; - if ((ri2 = digest256map_get(by_ed_key, pk))) { - /* Duplicate; must omit one. Set the omit_from_vote flag in whichever - * one has the earlier published_on. */ - const time_t ri_pub = ri->cache_info.published_on; - const time_t ri2_pub = ri2->cache_info.published_on; - if (ri2_pub < ri_pub || - (ri2_pub == ri_pub && - fast_memcmp(ri->cache_info.signed_descriptor_digest, - ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { - digest256map_set(by_ed_key, pk, ri); - ri2->omit_from_vote = 1; - } else { - ri->omit_from_vote = 1; - } - } else { - /* Add to map */ - digest256map_set(by_ed_key, pk, ri); - } - } SMARTLIST_FOREACH_END(ri); - - digest256map_free(by_ed_key, NULL); - - /* Now remove every router where the omit_from_vote flag got set. */ - SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { - if (ri->omit_from_vote) { - SMARTLIST_DEL_CURRENT(routers, ri); - } - } SMARTLIST_FOREACH_END(ri); -} - /** Extract status information from <b>ri</b> and from other authority - * functions and store it in <b>rs</b>>. + * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is + * set. * * We assume that ri-\>is_running has already been set, e.g. by * dirserv_set_router_is_running(ri, now); @@ -2249,6 +2053,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_valid = node->is_valid; if (node->is_fast && node->is_stable && + ri->supports_tunnelled_dir_requests && ((options->AuthDirGuardBWGuarantee && routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) || routerbw_kb >= MIN(guard_bandwidth_including_exits_kb, @@ -2284,6 +2089,9 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, OR port and it's reachable so copy it to the routerstatus. */ tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr); rs->ipv6_orport = ri->ipv6_orport; + } else { + tor_addr_make_null(&rs->ipv6_addr, AF_INET6); + rs->ipv6_orport = 0; } if (options->TestingTorNetwork) { @@ -2324,25 +2132,6 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs) } } -/** Routerstatus <b>rs</b> is part of a group of routers that are on - * too narrow an IP-space. Clear out its flags: we don't want people - * using it. - * - * Leave its BadExit flag alone though, since if we think it's a bad exit, - * we want to vote that way in case all the other authorities are voting - * Running and Exit. - */ -static void -clear_status_flags_on_sybil(routerstatus_t *rs) -{ - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_named = rs->is_valid = - rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0; - /* FFFF we might want some mechanism to check later on if we - * missed zeroing any flags: it's easy to add a new flag but - * forget to add it to this clause. */ -} - /** The guardfraction of the guard with identity fingerprint <b>guard_id</b> * is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for * this guard in <b>vote_routerstatuses</b>, and if we do, register the @@ -2659,17 +2448,38 @@ dirserv_read_guardfraction_file(const char *fname, /** * Helper function to parse out a line in the measured bandwidth file - * into a measured_bw_line_t output structure. Returns -1 on failure - * or 0 on success. + * into a measured_bw_line_t output structure. + * + * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete + * bw line, return -1 and warn, since we are after the headers and we should + * only parse bw lines. Return 0 otherwise. + * + * If <b>line_is_after_headers</b> is false then it means that we are not past + * the header block yet. If we encounter an incomplete bw line, return -1 but + * don't warn since there could be additional header lines coming. If we + * encounter a proper bw line, return 0 (and we got past the headers). */ STATIC int -measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line, + int line_is_after_headers) { char *line = tor_strdup(orig_line); char *cp = line; int got_bw = 0; int got_node_id = 0; char *strtok_state; /* lame sauce d'jour */ + + if (strlen(line) == 0) { + log_warn(LD_DIRSERV, "Empty line in bandwidth file"); + tor_free(line); + return -1; + } + + /* Remove end of line character, so that is not part of the token */ + if (line[strlen(line) - 1] == '\n') { + line[strlen(line) - 1] = '\0'; + } + cp = tor_strtok_r(cp, " \t", &strtok_state); if (!cp) { @@ -2687,7 +2497,13 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) } do { - if (strcmpstart(cp, "bw=") == 0) { + // If the line contains vote=0, ignore it. + if (strcmpstart(cp, "vote=0") == 0) { + log_debug(LD_DIRSERV, "Ignoring bandwidth file line that contains " + "vote=0: %s",escaped(orig_line)); + tor_free(line); + return -1; + } else if (strcmpstart(cp, "bw=") == 0) { int parse_ok = 0; char *endptr; if (got_bw) { @@ -2698,7 +2514,7 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) } cp+=strlen("bw="); - out->bw_kb = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + out->bw_kb = tor_parse_long(cp, 10, 0, LONG_MAX, &parse_ok, &endptr); if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", escaped(orig_line)); @@ -2731,6 +2547,13 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) if (got_bw && got_node_id) { tor_free(line); return 0; + } else if (line_is_after_headers == 0) { + /* There could be additional header lines, therefore do not give warnings + * but returns -1 since it's not a complete bw line. */ + log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; } else { log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", escaped(orig_line)); @@ -2779,6 +2602,11 @@ dirserv_read_measured_bandwidths(const char *from_file, int applied_lines = 0; time_t file_time, now; int ok; + /* This flag will be 1 only when the first successful bw measurement line + * has been encountered, so that measured_bw_line_parse don't give warnings + * if there are additional header lines, as introduced in Bandwidth List spec + * version 1.1.0 */ + int line_is_after_headers = 0; /* Initialise line, so that we can't possibly run off the end. */ memset(line, 0, sizeof(line)); @@ -2826,7 +2654,11 @@ dirserv_read_measured_bandwidths(const char *from_file, while (!feof(fp)) { measured_bw_line_t parsed_line; if (fgets(line, sizeof(line), fp) && strlen(line)) { - if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_parse(&parsed_line, line, + line_is_after_headers) != -1) { + /* This condition will be true when the first complete valid bw line + * has been encountered, which means the end of the header lines. */ + line_is_after_headers = 1; /* Also cache the line for dirserv_get_bandwidth_for_router() */ dirserv_cache_measured_bw(&parsed_line, file_time); if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) @@ -2845,346 +2677,67 @@ dirserv_read_measured_bandwidths(const char *from_file, return 0; } -/** Return a new networkstatus_t* containing our current opinion. (For v3 - * authorities) */ -networkstatus_t * -dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, - authority_cert_t *cert) -{ - const or_options_t *options = get_options(); - networkstatus_t *v3_out = NULL; - uint32_t addr; - char *hostname = NULL, *client_versions = NULL, *server_versions = NULL; - const char *contact; - smartlist_t *routers, *routerstatuses; - char identity_digest[DIGEST_LEN]; - char signing_key_digest[DIGEST_LEN]; - int listbadexits = options->AuthDirListBadExits; - routerlist_t *rl = router_get_routerlist(); - time_t now = time(NULL); - time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - networkstatus_voter_info_t *voter = NULL; - vote_timing_t timing; - digestmap_t *omit_as_sybil = NULL; - const int vote_on_reachability = running_long_enough_to_decide_unreachable(); - smartlist_t *microdescriptors = NULL; - - tor_assert(private_key); - tor_assert(cert); - - if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { - log_err(LD_BUG, "Error computing signing key digest"); - return NULL; - } - if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) { - log_err(LD_BUG, "Error computing identity key digest"); - return NULL; - } - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - return NULL; - } - if (!hostname || !strchr(hostname, '.')) { - tor_free(hostname); - hostname = tor_dup_ip(addr); - } - - if (options->VersioningAuthoritativeDir) { - client_versions = - format_recommended_version_list(options->RecommendedClientVersions, 0); - server_versions = - format_recommended_version_list(options->RecommendedServerVersions, 0); - } - - contact = get_options()->ContactInfo; - if (!contact) - contact = "(none)"; - - /* - * Do this so dirserv_compute_performance_thresholds() and - * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. - */ - if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); - } else { - /* - * No bandwidths file; clear the measured bandwidth cache in case we had - * one last time around. - */ - if (dirserv_get_measured_bw_cache_size() > 0) { - dirserv_clear_measured_bw_cache(); - } - } - - /* precompute this part, since we need it to decide what "stable" - * means. */ - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - dirserv_set_router_is_running(ri, now); - }); - - routers = smartlist_new(); - smartlist_add_all(routers, rl->routers); - routers_make_ed_keys_unique(routers); - /* After this point, don't use rl->routers; use 'routers' instead. */ - routers_sort_by_identity(routers); - omit_as_sybil = get_possible_sybil_list(routers); - - DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) { - (void) ignore; - rep_hist_make_router_pessimal(sybil_id, now); - } DIGESTMAP_FOREACH_END; - - /* Count how many have measured bandwidths so we know how to assign flags; - * this must come before dirserv_compute_performance_thresholds() */ - dirserv_count_measured_bws(routers); - - dirserv_compute_performance_thresholds(omit_as_sybil); - - routerstatuses = smartlist_new(); - microdescriptors = smartlist_new(); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - if (ri->cache_info.published_on >= cutoff) { - routerstatus_t *rs; - vote_routerstatus_t *vrs; - node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); - if (!node) - continue; - - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - set_routerstatus_from_routerinfo(rs, node, ri, now, - listbadexits); - - if (ri->cache_info.signing_key_cert) { - memcpy(vrs->ed25519_id, - ri->cache_info.signing_key_cert->signing_key.pubkey, - ED25519_PUBKEY_LEN); - } - - if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) - clear_status_flags_on_sybil(rs); - - if (!vote_on_reachability) - rs->is_flagged_running = 0; - - vrs->version = version_from_platform(ri->platform); - if (ri->protocol_list) { - vrs->protocols = tor_strdup(ri->protocol_list); - } else { - vrs->protocols = tor_strdup( - protover_compute_for_old_tor(vrs->version)); - } - vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now, - microdescriptors); - - smartlist_add(routerstatuses, vrs); - } - } SMARTLIST_FOREACH_END(ri); - - { - smartlist_t *added = - microdescs_add_list_to_cache(get_microdesc_cache(), - microdescriptors, SAVED_NOWHERE, 0); - smartlist_free(added); - smartlist_free(microdescriptors); - } - - smartlist_free(routers); - digestmap_free(omit_as_sybil, NULL); - - /* Apply guardfraction information to routerstatuses. */ - if (options->GuardfractionFile) { - dirserv_read_guardfraction_file(options->GuardfractionFile, - routerstatuses); - } - - /* This pass through applies the measured bw lines to the routerstatuses */ - if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, - routerstatuses); - } else { - /* - * No bandwidths file; clear the measured bandwidth cache in case we had - * one last time around. - */ - if (dirserv_get_measured_bw_cache_size() > 0) { - dirserv_clear_measured_bw_cache(); - } - } - - v3_out = tor_malloc_zero(sizeof(networkstatus_t)); - - v3_out->type = NS_TYPE_VOTE; - dirvote_get_preferred_voting_intervals(&timing); - v3_out->published = now; - { - char tbuf[ISO_TIME_LEN+1]; - networkstatus_t *current_consensus = - networkstatus_get_live_consensus(now); - long last_consensus_interval; /* only used to pick a valid_after */ - if (current_consensus) - last_consensus_interval = current_consensus->fresh_until - - current_consensus->valid_after; - else - last_consensus_interval = options->TestingV3AuthInitialVotingInterval; - v3_out->valid_after = - dirvote_get_start_of_next_interval(now, (int)last_consensus_interval, - options->TestingV3AuthVotingStartOffset); - format_iso_time(tbuf, v3_out->valid_after); - log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " - "consensus_set=%d, last_interval=%d", - tbuf, current_consensus?1:0, (int)last_consensus_interval); - } - v3_out->fresh_until = v3_out->valid_after + timing.vote_interval; - v3_out->valid_until = v3_out->valid_after + - (timing.vote_interval * timing.n_intervals_valid); - v3_out->vote_seconds = timing.vote_delay; - v3_out->dist_seconds = timing.dist_delay; - tor_assert(v3_out->vote_seconds > 0); - tor_assert(v3_out->dist_seconds > 0); - tor_assert(timing.n_intervals_valid > 0); - - v3_out->client_versions = client_versions; - v3_out->server_versions = server_versions; - - /* These are hardwired, to avoid disaster. */ - v3_out->recommended_relay_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2"); - v3_out->recommended_client_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2"); - v3_out->required_client_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2"); - v3_out->required_relay_protocols = - tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2"); - - /* We are not allowed to vote to require anything we don't have. */ - tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL)); - tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL)); - - /* We should not recommend anything we don't have. */ - tor_assert_nonfatal(protover_all_supported( - v3_out->recommended_relay_protocols, NULL)); - tor_assert_nonfatal(protover_all_supported( - v3_out->recommended_client_protocols, NULL)); - - v3_out->package_lines = smartlist_new(); - { - config_line_t *cl; - for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { - if (validate_recommended_package_line(cl->value)) - smartlist_add(v3_out->package_lines, tor_strdup(cl->value)); - } - } - - v3_out->known_flags = smartlist_new(); - smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (vote_on_reachability) - smartlist_add(v3_out->known_flags, tor_strdup("Running")); - if (listbadexits) - smartlist_add(v3_out->known_flags, tor_strdup("BadExit")); - smartlist_sort_strings(v3_out->known_flags); - - if (options->ConsensusParams) { - v3_out->net_params = smartlist_new(); - smartlist_split_string(v3_out->net_params, - options->ConsensusParams, NULL, 0, 0); - smartlist_sort_strings(v3_out->net_params); - } - - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup(options->Nickname); - memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); - voter->sigs = smartlist_new(); - voter->address = hostname; - voter->addr = addr; - voter->dir_port = router_get_advertised_dir_port(options, 0); - voter->or_port = router_get_advertised_or_port(options); - voter->contact = tor_strdup(contact); - if (options->V3AuthUseLegacyKey) { - authority_cert_t *c = get_my_v3_legacy_cert(); - if (c) { - if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) { - log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key"); - memset(voter->legacy_id_digest, 0, DIGEST_LEN); - } - } - } - - v3_out->voters = smartlist_new(); - smartlist_add(v3_out->voters, voter); - v3_out->cert = authority_cert_dup(cert); - v3_out->routerstatus_list = routerstatuses; - /* Note: networkstatus_digest is unset; it won't get set until we actually - * format the vote. */ - - return v3_out; -} - /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t * pointers, adds copies of digests to fps_out, and doesn't use the * /tor/server/ prefix. For a /d/ request, adds descriptor digests; for other * requests, adds identity digests. */ int -dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, - const char **msg, int for_unencrypted_conn, - int is_extrainfo) +dirserv_get_routerdesc_spool(smartlist_t *spool_out, + const char *key, + dir_spool_source_t source, + int conn_is_encrypted, + const char **msg_out) { - int by_id = 1; - *msg = NULL; + *msg_out = NULL; if (!strcmp(key, "all")) { - routerlist_t *rl = router_get_routerlist(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, - smartlist_add(fps_out, - tor_memdup(r->cache_info.identity_digest, DIGEST_LEN))); - /* Treat "all" requests as if they were unencrypted */ - for_unencrypted_conn = 1; + const routerlist_t *rl = router_get_routerlist(); + SMARTLIST_FOREACH_BEGIN(rl->routers, const routerinfo_t *, r) { + spooled_resource_t *spooled; + spooled = spooled_resource_new(source, + (const uint8_t *)r->cache_info.identity_digest, + DIGEST_LEN); + /* Treat "all" requests as if they were unencrypted */ + conn_is_encrypted = 0; + smartlist_add(spool_out, spooled); + } SMARTLIST_FOREACH_END(r); } else if (!strcmp(key, "authority")) { const routerinfo_t *ri = router_get_my_routerinfo(); if (ri) - smartlist_add(fps_out, - tor_memdup(ri->cache_info.identity_digest, DIGEST_LEN)); + smartlist_add(spool_out, + spooled_resource_new(source, + (const uint8_t *)ri->cache_info.identity_digest, + DIGEST_LEN)); } else if (!strcmpstart(key, "d/")) { - by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, - DSR_HEX|DSR_SORT_UNIQ); + dir_split_resource_into_spoolable(key, source, spool_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, - DSR_HEX|DSR_SORT_UNIQ); + dir_split_resource_into_spoolable(key, source, spool_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { - *msg = "Key not recognized"; + *msg_out = "Not found"; return -1; } - if (for_unencrypted_conn) { + if (! conn_is_encrypted) { /* Remove anything that insists it not be sent unencrypted. */ - SMARTLIST_FOREACH_BEGIN(fps_out, char *, cp) { - const signed_descriptor_t *sd; - if (by_id) - sd = get_signed_descriptor_by_fp(cp,is_extrainfo,0); - else if (is_extrainfo) - sd = extrainfo_get_by_descriptor_digest(cp); - else - sd = router_get_by_descriptor_digest(cp); - if (sd && !sd->send_unencrypted) { - tor_free(cp); - SMARTLIST_DEL_CURRENT(fps_out, cp); - } - } SMARTLIST_FOREACH_END(cp); + SMARTLIST_FOREACH_BEGIN(spool_out, spooled_resource_t *, spooled) { + const uint8_t *body = NULL; + size_t bodylen = 0; + int r = spooled_resource_lookup_body(spooled, conn_is_encrypted, + &body, &bodylen, NULL); + if (r < 0 || body == NULL || bodylen == 0) { + SMARTLIST_DEL_CURRENT(spool_out, spooled); + spooled_resource_free(spooled); + } + } SMARTLIST_FOREACH_END(spooled); } - if (!smartlist_len(fps_out)) { - *msg = "Servers unavailable"; + if (!smartlist_len(spool_out)) { + *msg_out = "Servers unavailable"; return -1; } return 0; @@ -3280,7 +2833,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, void dirserv_orconn_tls_done(const tor_addr_t *addr, uint16_t or_port, - const char *digest_rcvd) + const char *digest_rcvd, + const ed25519_public_key_t *ed_id_rcvd) { node_t *node = NULL; tor_addr_port_t orport; @@ -3292,8 +2846,26 @@ dirserv_orconn_tls_done(const tor_addr_t *addr, node = node_get_mutable_by_id(digest_rcvd); if (node == NULL || node->ri == NULL) return; + ri = node->ri; + if (get_options()->AuthDirTestEd25519LinkKeys && + node_supports_ed25519_link_authentication(node, 1) && + ri->cache_info.signing_key_cert) { + /* We allow the node to have an ed25519 key if we haven't been told one in + * the routerinfo, but if we *HAVE* been told one in the routerinfo, it + * needs to match. */ + const ed25519_public_key_t *expected_id = + &ri->cache_info.signing_key_cert->signing_key; + tor_assert(!ed25519_public_key_is_zero(expected_id)); + if (! ed_id_rcvd || ! ed25519_pubkey_eq(ed_id_rcvd, expected_id)) { + log_info(LD_DIRSERV, "Router at %s:%d with RSA ID %s " + "did not present expected Ed25519 ID.", + fmt_addr(addr), or_port, hex_str(digest_rcvd, DIGEST_LEN)); + return; /* Don't mark it as reachable. */ + } + } + tor_addr_copy(&orport.addr, addr); orport.port = or_port; if (router_has_orport(ri, &orport)) { @@ -3301,7 +2873,7 @@ dirserv_orconn_tls_done(const tor_addr_t *addr, if (!authdir_mode_bridge(get_options()) || ri->purpose == ROUTER_PURPOSE_BRIDGE) { char addrstr[TOR_ADDR_BUF_LEN]; - /* This is a bridge or we're not a bridge authorititative -- + /* This is a bridge or we're not a bridge authority -- mark it as reachable. */ log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", router_describe(ri), @@ -3349,21 +2921,32 @@ dirserv_should_launch_reachability_test(const routerinfo_t *ri, void dirserv_single_reachability_test(time_t now, routerinfo_t *router) { + const or_options_t *options = get_options(); channel_t *chan = NULL; - node_t *node = NULL; + const node_t *node = NULL; tor_addr_t router_addr; + const ed25519_public_key_t *ed_id_key; (void) now; tor_assert(router); - node = node_get_mutable_by_id(router->cache_info.identity_digest); + node = node_get_by_id(router->cache_info.identity_digest); tor_assert(node); + if (options->AuthDirTestEd25519LinkKeys && + node_supports_ed25519_link_authentication(node, 1) && + router->cache_info.signing_key_cert) { + ed_id_key = &router->cache_info.signing_key_cert->signing_key; + } else { + ed_id_key = NULL; + } + /* IPv4. */ log_debug(LD_OR,"Testing reachability of %s at %s:%u.", router->nickname, fmt_addr32(router->addr), router->or_port); tor_addr_from_ipv4h(&router_addr, router->addr); chan = channel_tls_connect(&router_addr, router->or_port, - router->cache_info.identity_digest); + router->cache_info.identity_digest, + ed_id_key); if (chan) command_setup_channel(chan); /* Possible IPv6. */ @@ -3375,7 +2958,8 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router) tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1), router->ipv6_orport); chan = channel_tls_connect(&router->ipv6_addr, router->ipv6_orport, - router->cache_info.identity_digest); + router->cache_info.identity_digest, + ed_id_key); if (chan) command_setup_channel(chan); } } @@ -3418,410 +3002,502 @@ dirserv_test_reachability(time_t now) ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */ } -/** Given a fingerprint <b>fp</b> which is either set if we're looking for a - * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded - * flavor name if we want a flavored v3 status, return a pointer to the - * appropriate cached dir object, or NULL if there isn't one available. */ -static cached_dir_t * -lookup_cached_dir_by_fp(const char *fp) +/* ========== + * Spooling code. + * ========== */ + +spooled_resource_t * +spooled_resource_new(dir_spool_source_t source, + const uint8_t *digest, size_t digestlen) { - cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_consensuses) { - d = strmap_get(cached_consensuses, "ns"); - } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && - (d = strmap_get(cached_consensuses, fp))) { - /* this here interface is a nasty hack XXXX */; + spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t)); + spooled->spool_source = source; + switch (source) { + case DIR_SPOOL_NETWORKSTATUS: + spooled->spool_eagerly = 0; + break; + case DIR_SPOOL_SERVER_BY_DIGEST: + case DIR_SPOOL_SERVER_BY_FP: + case DIR_SPOOL_EXTRA_BY_DIGEST: + case DIR_SPOOL_EXTRA_BY_FP: + case DIR_SPOOL_MICRODESC: + default: + spooled->spool_eagerly = 1; + break; + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: + tor_assert_unreached(); + break; } - return d; + tor_assert(digestlen <= sizeof(spooled->digest)); + if (digest) + memcpy(spooled->digest, digest, digestlen); + return spooled; } -/** Remove from <b>fps</b> every networkstatus key where both - * a) we have a networkstatus document and - * b) it is not newer than <b>cutoff</b>. +/** + * Create a new spooled_resource_t to spool the contents of <b>entry</b> to + * the user. Return the spooled object on success, or NULL on failure (which + * is probably caused by a failure to map the body of the item from disk). * - * Return 1 if any items were present at all; else return 0. + * Adds a reference to entry's reference counter. */ -int -dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff) -{ - int found_any = 0; - SMARTLIST_FOREACH_BEGIN(fps, char *, digest) { - cached_dir_t *d = lookup_cached_dir_by_fp(digest); - if (!d) - continue; - found_any = 1; - if (d->published <= cutoff) { - tor_free(digest); - SMARTLIST_DEL_CURRENT(fps, digest); - } - } SMARTLIST_FOREACH_END(digest); - - return found_any; +spooled_resource_t * +spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry) +{ + spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t)); + spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY; + spooled->spool_eagerly = 0; + consensus_cache_entry_incref(entry); + spooled->consensus_cache_entry = entry; + + int r = consensus_cache_entry_get_body(entry, + &spooled->cce_body, + &spooled->cce_len); + if (r == 0) { + return spooled; + } else { + spooled_resource_free(spooled); + return NULL; + } } -/** Return the cache-info for identity fingerprint <b>fp</b>, or - * its extra-info document if <b>extrainfo</b> is true. Return - * NULL if not found or if the descriptor is older than - * <b>publish_cutoff</b>. */ -static const signed_descriptor_t * -get_signed_descriptor_by_fp(const char *fp, int extrainfo, - time_t publish_cutoff) +/** Release all storage held by <b>spooled</b>. */ +void +spooled_resource_free_(spooled_resource_t *spooled) { - if (router_digest_is_me(fp)) { - if (extrainfo) - return &(router_get_my_extrainfo()->cache_info); - else - return &(router_get_my_routerinfo()->cache_info); - } else { - const routerinfo_t *ri = router_get_by_id_digest(fp); - if (ri && - ri->cache_info.published_on > publish_cutoff) { - if (extrainfo) - return extrainfo_get_by_descriptor_digest( - ri->cache_info.extra_info_digest); - else - return &ri->cache_info; - } + if (spooled == NULL) + return; + + if (spooled->cached_dir_ref) { + cached_dir_decref(spooled->cached_dir_ref); } - return NULL; -} -/** Return true iff we have any of the documents (extrainfo or routerdesc) - * specified by the fingerprints in <b>fps</b> and <b>spool_src</b>. Used to - * decide whether to send a 404. */ -int -dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) -{ - time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH; - SMARTLIST_FOREACH_BEGIN(fps, const char *, fp) { - switch (spool_src) - { - case DIR_SPOOL_EXTRA_BY_DIGEST: - if (extrainfo_get_by_descriptor_digest(fp)) return 1; - break; - case DIR_SPOOL_SERVER_BY_DIGEST: - if (router_get_by_descriptor_digest(fp)) return 1; - break; - case DIR_SPOOL_EXTRA_BY_FP: - case DIR_SPOOL_SERVER_BY_FP: - if (get_signed_descriptor_by_fp(fp, - spool_src == DIR_SPOOL_EXTRA_BY_FP, publish_cutoff)) - return 1; - break; - } - } SMARTLIST_FOREACH_END(fp); - return 0; + if (spooled->consensus_cache_entry) { + consensus_cache_entry_decref(spooled->consensus_cache_entry); + } + + tor_free(spooled); } -/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of - * a microdescriptor we have. */ -int -dirserv_have_any_microdesc(const smartlist_t *fps) +/** When spooling data from a cached_dir_t object, we always add + * at least this much. */ +#define DIRSERV_CACHED_DIR_CHUNK_SIZE 8192 + +/** Return an compression ratio for compressing objects from <b>source</b>. + */ +static double +estimate_compression_ratio(dir_spool_source_t source) { - microdesc_cache_t *cache = get_microdesc_cache(); - SMARTLIST_FOREACH(fps, const char *, fp, - if (microdesc_cache_lookup_by_digest256(cache, fp)) - return 1); - return 0; + /* We should put in better estimates here, depending on the number of + objects and their type */ + (void) source; + return 0.5; } -/** Return an approximate estimate of the number of bytes that will - * be needed to transmit the server descriptors (if is_serverdescs -- - * they can be either d/ or fp/ queries) or networkstatus objects (if - * !is_serverdescs) listed in <b>fps</b>. If <b>compressed</b> is set, - * we guess how large the data will be after compression. +/** Return an estimated number of bytes needed for transmitting the + * resource in <b>spooled</b> on <b>conn</b> * - * The return value is an estimate; it might be larger or smaller. - **/ -size_t -dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, - int compressed) -{ - size_t result; - tor_assert(fps); - if (is_serverdescs) { - int n = smartlist_len(fps); - const routerinfo_t *me = router_get_my_routerinfo(); - result = (me?me->cache_info.signed_descriptor_len:2048) * n; - if (compressed) - result /= 2; /* observed compressibility is between 35 and 55%. */ + * As a convenient side-effect, set *<b>published_out</b> to the resource's + * publication time. + */ +static size_t +spooled_resource_estimate_size(const spooled_resource_t *spooled, + dir_connection_t *conn, + int compressed, + time_t *published_out) +{ + if (spooled->spool_eagerly) { + const uint8_t *body = NULL; + size_t bodylen = 0; + int r = spooled_resource_lookup_body(spooled, + connection_dir_is_encrypted(conn), + &body, &bodylen, + published_out); + if (r == -1 || body == NULL || bodylen == 0) + return 0; + if (compressed) { + double ratio = estimate_compression_ratio(spooled->spool_source); + bodylen = (size_t)(bodylen * ratio); + } + return bodylen; } else { - result = 0; - SMARTLIST_FOREACH(fps, const char *, digest, { - cached_dir_t *dir = lookup_cached_dir_by_fp(digest); - if (dir) - result += compressed ? dir->dir_z_len : dir->dir_len; - }); + cached_dir_t *cached; + if (spooled->consensus_cache_entry) { + if (published_out) { + consensus_cache_entry_get_valid_after( + spooled->consensus_cache_entry, published_out); + } + + return spooled->cce_len; + } + if (spooled->cached_dir_ref) { + cached = spooled->cached_dir_ref; + } else { + cached = spooled_resource_lookup_cached_dir(spooled, + published_out); + } + if (cached == NULL) { + return 0; + } + size_t result = compressed ? cached->dir_compressed_len : cached->dir_len; + return result; } - return result; } -/** Given a list of microdescriptor hashes, guess how many bytes will be - * needed to transmit them, and return the guess. */ -size_t -dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) -{ - size_t result = smartlist_len(fps) * microdesc_average_size(NULL); - if (compressed) - result /= 2; - return result; -} +/** Return code for spooled_resource_flush_some */ +typedef enum { + SRFS_ERR = -1, + SRFS_MORE = 0, + SRFS_DONE +} spooled_resource_flush_status_t; -/** When we're spooling data onto our outbuf, add more whenever we dip - * below this threshold. */ -#define DIRSERV_BUFFER_MIN 16384 +/** Flush some or all of the bytes from <b>spooled</b> onto <b>conn</b>. + * Return SRFS_ERR on error, SRFS_MORE if there are more bytes to flush from + * this spooled resource, or SRFS_DONE if we are done flushing this spooled + * resource. + */ +static spooled_resource_flush_status_t +spooled_resource_flush_some(spooled_resource_t *spooled, + dir_connection_t *conn) +{ + if (spooled->spool_eagerly) { + /* Spool_eagerly resources are sent all-at-once. */ + const uint8_t *body = NULL; + size_t bodylen = 0; + int r = spooled_resource_lookup_body(spooled, + connection_dir_is_encrypted(conn), + &body, &bodylen, NULL); + if (r == -1 || body == NULL || bodylen == 0) { + /* Absent objects count as "done". */ + return SRFS_DONE; + } + if (conn->compress_state) { + connection_buf_add_compress((const char*)body, bodylen, conn, 0); + } else { + connection_buf_add((const char*)body, bodylen, TO_CONN(conn)); + } + return SRFS_DONE; + } else { + cached_dir_t *cached = spooled->cached_dir_ref; + consensus_cache_entry_t *cce = spooled->consensus_cache_entry; + if (cached == NULL && cce == NULL) { + /* The cached_dir_t hasn't been materialized yet. So let's look it up. */ + cached = spooled->cached_dir_ref = + spooled_resource_lookup_cached_dir(spooled, NULL); + if (!cached) { + /* Absent objects count as done. */ + return SRFS_DONE; + } + ++cached->refcnt; + tor_assert_nonfatal(spooled->cached_dir_offset == 0); + } -/** Spooling helper: called when we have no more data to spool to <b>conn</b>. - * Flushes any remaining data to be (un)compressed, and changes the spool - * source to NONE. Returns 0 on success, negative on failure. */ -static int -connection_dirserv_finish_spooling(dir_connection_t *conn) -{ - if (conn->zlib_state) { - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; + if (BUG(!cached && !cce)) + return SRFS_DONE; + + int64_t total_len; + const char *ptr; + if (cached) { + total_len = cached->dir_compressed_len; + ptr = cached->dir_compressed; + } else { + total_len = spooled->cce_len; + ptr = (const char *)spooled->cce_body; + } + /* How many bytes left to flush? */ + int64_t remaining; + remaining = total_len - spooled->cached_dir_offset; + if (BUG(remaining < 0)) + return SRFS_ERR; + ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); + if (conn->compress_state) { + connection_buf_add_compress( + ptr + spooled->cached_dir_offset, + bytes, conn, 0); + } else { + connection_buf_add(ptr + spooled->cached_dir_offset, + bytes, TO_CONN(conn)); + } + spooled->cached_dir_offset += bytes; + if (spooled->cached_dir_offset >= (off_t)total_len) { + return SRFS_DONE; + } else { + return SRFS_MORE; + } } - conn->dir_spool_src = DIR_SPOOL_NONE; - return 0; } -/** Spooling helper: called when we're sending a bunch of server descriptors, - * and the outbuf has become too empty. Pulls some entries from - * fingerprint_stack, and writes the corresponding servers onto outbuf. If we - * run out of entries, flushes the zlib state and sets the spool source to - * NONE. Returns 0 on success, negative on failure. +/** Helper: find the cached_dir_t for a spooled_resource_t, for + * sending it to <b>conn</b>. Set *<b>published_out</b>, if provided, + * to the published time of the cached_dir_t. + * + * DOES NOT increase the reference count on the result. Callers must do that + * themselves if they mean to hang on to it. */ +static cached_dir_t * +spooled_resource_lookup_cached_dir(const spooled_resource_t *spooled, + time_t *published_out) +{ + tor_assert(spooled->spool_eagerly == 0); + cached_dir_t *d = lookup_cached_dir_by_fp(spooled->digest); + if (d != NULL) { + if (published_out) + *published_out = d->published; + } + return d; +} + +/** Helper: Look up the body for an eagerly-served spooled_resource. If + * <b>conn_is_encrypted</b> is false, don't look up any resource that + * shouldn't be sent over an unencrypted connection. On success, set + * <b>body_out</b>, <b>size_out</b>, and <b>published_out</b> to refer + * to the resource's body, size, and publication date, and return 0. + * On failure return -1. */ static int -connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) +spooled_resource_lookup_body(const spooled_resource_t *spooled, + int conn_is_encrypted, + const uint8_t **body_out, + size_t *size_out, + time_t *published_out) { - int by_fp = (conn->dir_spool_src == DIR_SPOOL_SERVER_BY_FP || - conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP); - int extra = (conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP || - conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_DIGEST); - time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH; + tor_assert(spooled->spool_eagerly == 1); - const or_options_t *options = get_options(); + const signed_descriptor_t *sd = NULL; - while (smartlist_len(conn->fingerprint_stack) && - connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { - const char *body; - char *fp = smartlist_pop_last(conn->fingerprint_stack); - const signed_descriptor_t *sd = NULL; - if (by_fp) { - sd = get_signed_descriptor_by_fp(fp, extra, publish_cutoff); - } else { - sd = extra ? extrainfo_get_by_descriptor_digest(fp) - : router_get_by_descriptor_digest(fp); + switch (spooled->spool_source) { + case DIR_SPOOL_EXTRA_BY_FP: { + sd = get_signed_descriptor_by_fp(spooled->digest, 1); + break; } - tor_free(fp); - if (!sd) - continue; - if (!connection_dir_is_encrypted(conn) && !sd->send_unencrypted) { - /* we did this check once before (so we could have an accurate size - * estimate and maybe send a 404 if somebody asked for only bridges on a - * connection), but we need to do it again in case a previously - * unknown bridge descriptor has shown up between then and now. */ - continue; + case DIR_SPOOL_SERVER_BY_FP: { + sd = get_signed_descriptor_by_fp(spooled->digest, 0); + break; } - - /** If we are the bridge authority and the descriptor is a bridge - * descriptor, remember that we served this descriptor for desc stats. */ - if (options->BridgeAuthoritativeDir && by_fp) { - const routerinfo_t *router = - router_get_by_id_digest(sd->identity_digest); - /* router can be NULL here when the bridge auth is asked for its own - * descriptor. */ - if (router && router->purpose == ROUTER_PURPOSE_BRIDGE) - rep_hist_note_desc_served(sd->identity_digest); + case DIR_SPOOL_SERVER_BY_DIGEST: { + sd = router_get_by_descriptor_digest((const char *)spooled->digest); + break; } - body = signed_descriptor_get_body(sd); - if (conn->zlib_state) { - int last = ! smartlist_len(conn->fingerprint_stack); - connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, - last); - if (last) { - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; + case DIR_SPOOL_EXTRA_BY_DIGEST: { + sd = extrainfo_get_by_descriptor_digest((const char *)spooled->digest); + break; + } + case DIR_SPOOL_MICRODESC: { + microdesc_t *md = microdesc_cache_lookup_by_digest256( + get_microdesc_cache(), + (const char *)spooled->digest); + if (! md || ! md->body) { + return -1; } - } else { - connection_write_to_buf(body, - sd->signed_descriptor_len, - TO_CONN(conn)); + *body_out = (const uint8_t *)md->body; + *size_out = md->bodylen; + if (published_out) + *published_out = TIME_MAX; + return 0; } + case DIR_SPOOL_NETWORKSTATUS: + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + return -1; + /* LCOV_EXCL_STOP */ } - if (!smartlist_len(conn->fingerprint_stack)) { - /* We just wrote the last one; finish up. */ - if (conn->zlib_state) { - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; - } - conn->dir_spool_src = DIR_SPOOL_NONE; - smartlist_free(conn->fingerprint_stack); - conn->fingerprint_stack = NULL; + /* If we get here, then we tried to set "sd" to a signed_descriptor_t. */ + + if (sd == NULL) { + return -1; } + if (sd->send_unencrypted == 0 && ! conn_is_encrypted) { + /* we did this check once before (so we could have an accurate size + * estimate and maybe send a 404 if somebody asked for only bridges on + * a connection), but we need to do it again in case a previously + * unknown bridge descriptor has shown up between then and now. */ + return -1; + } + *body_out = (const uint8_t *) signed_descriptor_get_body(sd); + *size_out = sd->signed_descriptor_len; + if (published_out) + *published_out = sd->published_on; return 0; } -/** Spooling helper: called when we're sending a bunch of microdescriptors, - * and the outbuf has become too empty. Pulls some entries from - * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If - * we run out of entries, flushes the zlib state and sets the spool source to - * NONE. Returns 0 on success, negative on failure. - */ -static int -connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) -{ - microdesc_cache_t *cache = get_microdesc_cache(); - while (smartlist_len(conn->fingerprint_stack) && - connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { - char *fp256 = smartlist_pop_last(conn->fingerprint_stack); - microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); - tor_free(fp256); - if (!md || !md->body) - continue; - if (conn->zlib_state) { - int last = !smartlist_len(conn->fingerprint_stack); - connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); - if (last) { - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; - } - } else { - connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); - } - } - if (!smartlist_len(conn->fingerprint_stack)) { - if (conn->zlib_state) { - connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); - conn->zlib_state = NULL; - } - conn->dir_spool_src = DIR_SPOOL_NONE; - smartlist_free(conn->fingerprint_stack); - conn->fingerprint_stack = NULL; +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ +static cached_dir_t * +lookup_cached_dir_by_fp(const uint8_t *fp) +{ + cached_dir_t *d = NULL; + if (tor_digest_is_zero((const char *)fp) && cached_consensuses) { + d = strmap_get(cached_consensuses, "ns"); + } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses) { + /* this here interface is a nasty hack: we're shoving a flavor into + * a digest field. */ + d = strmap_get(cached_consensuses, (const char *)fp); } - return 0; + return d; } -/** Spooling helper: Called when we're sending a directory or networkstatus, - * and the outbuf has become too empty. Pulls some bytes from - * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and - * puts them on the outbuf. If we run out of entries, flushes the zlib state - * and sets the spool source to NONE. Returns 0 on success, negative on - * failure. */ -static int -connection_dirserv_add_dir_bytes_to_outbuf(dir_connection_t *conn) -{ - ssize_t bytes; - int64_t remaining; - - bytes = DIRSERV_BUFFER_MIN - connection_get_outbuf_len(TO_CONN(conn)); - tor_assert(bytes > 0); - tor_assert(conn->cached_dir); - if (bytes < 8192) - bytes = 8192; - remaining = conn->cached_dir->dir_z_len - conn->cached_dir_offset; - if (bytes > remaining) - bytes = (ssize_t) remaining; - - if (conn->zlib_state) { - connection_write_to_buf_zlib( - conn->cached_dir->dir_z + conn->cached_dir_offset, - bytes, conn, bytes == remaining); - } else { - connection_write_to_buf(conn->cached_dir->dir_z + conn->cached_dir_offset, - bytes, TO_CONN(conn)); +/** Try to guess the number of bytes that will be needed to send the + * spooled objects for <b>conn</b>'s outgoing spool. In the process, + * remove every element of the spool that refers to an absent object, or + * which was published earlier than <b>cutoff</b>. Set *<b>size_out</b> + * to the number of bytes, and *<b>n_expired_out</b> to the number of + * objects removed for being too old. */ +void +dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, + time_t cutoff, + int compression, + size_t *size_out, + int *n_expired_out) +{ + if (BUG(!conn)) + return; + + smartlist_t *spool = conn->spool; + if (!spool) { + if (size_out) + *size_out = 0; + if (n_expired_out) + *n_expired_out = 0; + return; } - conn->cached_dir_offset += bytes; - if (conn->cached_dir_offset == (int)conn->cached_dir->dir_z_len) { - /* We just wrote the last one; finish up. */ - connection_dirserv_finish_spooling(conn); - cached_dir_decref(conn->cached_dir); - conn->cached_dir = NULL; + int n_expired = 0; + uint64_t total = 0; + SMARTLIST_FOREACH_BEGIN(spool, spooled_resource_t *, spooled) { + time_t published = TIME_MAX; + size_t sz = spooled_resource_estimate_size(spooled, conn, + compression, &published); + if (published < cutoff) { + ++n_expired; + SMARTLIST_DEL_CURRENT(spool, spooled); + spooled_resource_free(spooled); + } else if (sz == 0) { + SMARTLIST_DEL_CURRENT(spool, spooled); + spooled_resource_free(spooled); + } else { + total += sz; + } + } SMARTLIST_FOREACH_END(spooled); + + if (size_out) { + *size_out = (total > SIZE_MAX) ? SIZE_MAX : (size_t)total; } - return 0; + if (n_expired_out) + *n_expired_out = n_expired; } -/** Spooling helper: Called when we're spooling networkstatus objects on - * <b>conn</b>, and the outbuf has become too empty. If the current - * networkstatus object (in <b>conn</b>-\>cached_dir) has more data, pull data - * from there. Otherwise, pop the next fingerprint from fingerprint_stack, - * and start spooling the next networkstatus. (A digest of all 0 bytes is - * treated as a request for the current consensus.) If we run out of entries, - * flushes the zlib state and sets the spool source to NONE. Returns 0 on - * success, negative on failure. */ +/** Helper: used to sort a connection's spool. */ static int -connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) -{ - - while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { - if (conn->cached_dir) { - int uncompressing = (conn->zlib_state != NULL); - int r = connection_dirserv_add_dir_bytes_to_outbuf(conn); - if (conn->dir_spool_src == DIR_SPOOL_NONE) { - /* add_dir_bytes thinks we're done with the cached_dir. But we - * may have more cached_dirs! */ - conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS; - /* This bit is tricky. If we were uncompressing the last - * networkstatus, we may need to make a new zlib object to - * uncompress the next one. */ - if (uncompressing && ! conn->zlib_state && - conn->fingerprint_stack && - smartlist_len(conn->fingerprint_stack)) { - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); - } - } - if (r) return r; - } else if (conn->fingerprint_stack && - smartlist_len(conn->fingerprint_stack)) { - /* Add another networkstatus; start serving it. */ - char *fp = smartlist_pop_last(conn->fingerprint_stack); - cached_dir_t *d = lookup_cached_dir_by_fp(fp); - tor_free(fp); - if (d) { - ++d->refcnt; - conn->cached_dir = d; - conn->cached_dir_offset = 0; - } - } else { - connection_dirserv_finish_spooling(conn); - smartlist_free(conn->fingerprint_stack); - conn->fingerprint_stack = NULL; - return 0; +dirserv_spool_sort_comparison_(const void **a_, const void **b_) +{ + const spooled_resource_t *a = *a_; + const spooled_resource_t *b = *b_; + return fast_memcmp(a->digest, b->digest, sizeof(a->digest)); +} + +/** Sort all the entries in <b>conn</b> by digest. */ +void +dirserv_spool_sort(dir_connection_t *conn) +{ + if (conn->spool == NULL) + return; + smartlist_sort(conn->spool, dirserv_spool_sort_comparison_); +} + +/** Return the cache-info for identity fingerprint <b>fp</b>, or + * its extra-info document if <b>extrainfo</b> is true. Return + * NULL if not found or if the descriptor is older than + * <b>publish_cutoff</b>. */ +static const signed_descriptor_t * +get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo) +{ + if (router_digest_is_me((const char *)fp)) { + if (extrainfo) + return &(router_get_my_extrainfo()->cache_info); + else + return &(router_get_my_routerinfo()->cache_info); + } else { + const routerinfo_t *ri = router_get_by_id_digest((const char *)fp); + if (ri) { + if (extrainfo) + return extrainfo_get_by_descriptor_digest( + ri->cache_info.extra_info_digest); + else + return &ri->cache_info; } } - return 0; + return NULL; } -/** Called whenever we have flushed some directory data in state - * SERVER_WRITING. */ +/** When we're spooling data onto our outbuf, add more whenever we dip + * below this threshold. */ +#define DIRSERV_BUFFER_MIN 16384 + +/** + * Called whenever we have flushed some directory data in state + * SERVER_WRITING, or whenever we want to fill the buffer with initial + * directory data (so that subsequent writes will occur, and trigger this + * function again.) + * + * Return 0 on success, and -1 on failure. + */ int connection_dirserv_flushed_some(dir_connection_t *conn) { tor_assert(conn->base_.state == DIR_CONN_STATE_SERVER_WRITING); - - if (connection_get_outbuf_len(TO_CONN(conn)) >= DIRSERV_BUFFER_MIN) + if (conn->spool == NULL) return 0; - switch (conn->dir_spool_src) { - case DIR_SPOOL_EXTRA_BY_DIGEST: - case DIR_SPOOL_EXTRA_BY_FP: - case DIR_SPOOL_SERVER_BY_DIGEST: - case DIR_SPOOL_SERVER_BY_FP: - return connection_dirserv_add_servers_to_outbuf(conn); - case DIR_SPOOL_MICRODESC: - return connection_dirserv_add_microdescs_to_outbuf(conn); - case DIR_SPOOL_CACHED_DIR: - return connection_dirserv_add_dir_bytes_to_outbuf(conn); - case DIR_SPOOL_NETWORKSTATUS: - return connection_dirserv_add_networkstatus_bytes_to_outbuf(conn); - case DIR_SPOOL_NONE: - default: + while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN && + smartlist_len(conn->spool)) { + spooled_resource_t *spooled = + smartlist_get(conn->spool, smartlist_len(conn->spool)-1); + spooled_resource_flush_status_t status; + status = spooled_resource_flush_some(spooled, conn); + if (status == SRFS_ERR) { + return -1; + } else if (status == SRFS_MORE) { return 0; + } + tor_assert(status == SRFS_DONE); + + /* If we're here, we're done flushing this resource. */ + tor_assert(smartlist_pop_last(conn->spool) == spooled); + spooled_resource_free(spooled); + } + + if (smartlist_len(conn->spool) > 0) { + /* We're still spooling something. */ + return 0; + } + + /* If we get here, we're done. */ + smartlist_free(conn->spool); + conn->spool = NULL; + if (conn->compress_state) { + /* Flush the compression state: there could be more bytes pending in there, + * and we don't want to omit bytes. */ + connection_buf_add_compress("", 0, conn, 1); + tor_compress_free(conn->compress_state); + conn->compress_state = NULL; } + return 0; +} + +/** Remove every element from <b>conn</b>'s outgoing spool, and delete + * the spool. */ +void +dir_conn_clear_spool(dir_connection_t *conn) +{ + if (!conn || ! conn->spool) + return; + SMARTLIST_FOREACH(conn->spool, spooled_resource_t *, s, + spooled_resource_free(s)); + smartlist_free(conn->spool); + conn->spool = NULL; } /** Return true iff <b>line</b> is a valid RecommendedPackages line. |