diff options
| -rw-r--r-- | changes/bug16389 | 12 | ||||
| -rw-r--r-- | changes/bug16539 | 4 | ||||
| -rw-r--r-- | changes/bug16742 | 3 | ||||
| -rw-r--r-- | changes/decouple_dir_all_unreachable | 4 | ||||
| -rw-r--r-- | changes/decouple_init_keys | 3 | ||||
| -rw-r--r-- | changes/move_formatting_functions | 3 | ||||
| -rw-r--r-- | src/common/compat.c | 2 | ||||
| -rw-r--r-- | src/common/container.c | 6 | ||||
| -rw-r--r-- | src/common/container.h | 8 | ||||
| -rw-r--r-- | src/config/torrc.minimal.in-staging | 6 | ||||
| -rw-r--r-- | src/config/torrc.sample.in | 6 | ||||
| -rw-r--r-- | src/or/dirvote.c | 2 | ||||
| -rw-r--r-- | src/or/main.c | 50 | ||||
| -rw-r--r-- | src/or/rendcache.c | 302 | ||||
| -rw-r--r-- | src/or/rendcache.h | 23 | ||||
| -rw-r--r-- | src/or/rendclient.c | 7 | ||||
| -rw-r--r-- | src/or/rendclient.h | 4 | ||||
| -rw-r--r-- | src/or/rendcommon.h | 6 | ||||
| -rw-r--r-- | src/or/router.c | 68 | ||||
| -rw-r--r-- | src/or/router.h | 1 | ||||
| -rw-r--r-- | src/or/routerlist.c | 2 | ||||
| -rw-r--r-- | src/test/test.h | 9 | ||||
| -rw-r--r-- | src/test/test_containers.c | 153 | ||||
| -rw-r--r-- | src/test/test_util.c | 2 |
24 files changed, 626 insertions, 60 deletions
diff --git a/changes/bug16389 b/changes/bug16389 new file mode 100644 index 0000000000..b7eb35034a --- /dev/null +++ b/changes/bug16389 @@ -0,0 +1,12 @@ + o Hidden Service Enhancement + Client now uses an introduction point failure cache to know when to + fetch or keep a descriptor in their cache. + + When fetching a descriptor, for every introduction points in it, we look + them up in the failure cache to know if we keep the descriptor or not. + For this to work, everytime an introduction points is discarded (ex: + receiving a NACK), we note it down in our introduction cache. If all + introduction points for an onion service are in our failure cache, we + discard the descriptor and fetch a new one. + + See rendcache.c for a detailed explanation of the cache's behavior. diff --git a/changes/bug16539 b/changes/bug16539 new file mode 100644 index 0000000000..8a0b6d251c --- /dev/null +++ b/changes/bug16539 @@ -0,0 +1,4 @@ + o Minor bugfixes (Ed25519): + - Fix a memory leak when reading router descriptors with + expired Ed25519 certificate. Fixes bug 16539; bugfix on 0.2.7.2-alpha. + diff --git a/changes/bug16742 b/changes/bug16742 new file mode 100644 index 0000000000..2002cb7c72 --- /dev/null +++ b/changes/bug16742 @@ -0,0 +1,3 @@ + o Documentation: + - Recommend a 40 GB example AccountingMax in torrc.sample rather + than a 4 GB max. Closes ticket 16742. diff --git a/changes/decouple_dir_all_unreachable b/changes/decouple_dir_all_unreachable new file mode 100644 index 0000000000..1e57b3dfbd --- /dev/null +++ b/changes/decouple_dir_all_unreachable @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Simply the control graph further by deferring the inner body of + directory_all_unreachable() into a callback. Closes ticket + 16762.
\ No newline at end of file diff --git a/changes/decouple_init_keys b/changes/decouple_init_keys new file mode 100644 index 0000000000..7f48d2b9d3 --- /dev/null +++ b/changes/decouple_init_keys @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Move the client-only parts of init_keys() into a separate function. + Closes ticket 16763. diff --git a/changes/move_formatting_functions b/changes/move_formatting_functions new file mode 100644 index 0000000000..4ad5806f23 --- /dev/null +++ b/changes/move_formatting_functions @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Move some format-parsing functions out of crypto.c and + crypto_curve25519.c into crypto_format.c and/or util_format.c. diff --git a/src/common/compat.c b/src/common/compat.c index 10c43416bf..76f9bcb97e 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -3417,7 +3417,7 @@ tor_get_avail_disk_space(const char *path) if (!ok) { return -1; } - return (int64_t)freeBytesAvail; + return (int64_t)freeBytesAvail.QuadPart; #else (void)path; errno = ENOSYS; diff --git a/src/common/container.c b/src/common/container.c index 082afb51ee..636dfb6c57 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -742,7 +742,7 @@ smartlist_sort_strings(smartlist_t *sl) } /** Return the most frequent string in the sorted list <b>sl</b> */ -char * +const char * smartlist_get_most_frequent_string(smartlist_t *sl) { return smartlist_get_most_frequent(sl, compare_string_ptrs_); @@ -752,7 +752,7 @@ smartlist_get_most_frequent_string(smartlist_t *sl) * If <b>count_out</b> is provided, set <b>count_out</b> to the * number of times that string appears. */ -char * +const char * smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out) { return smartlist_get_most_frequent_(sl, compare_string_ptrs_, count_out); @@ -1020,7 +1020,7 @@ smartlist_sort_digests256(smartlist_t *sl) /** Return the most frequent member of the sorted list of DIGEST256_LEN * digests in <b>sl</b> */ -char * +const uint8_t * smartlist_get_most_frequent_digest256(smartlist_t *sl) { return smartlist_get_most_frequent(sl, compare_digests256_); diff --git a/src/common/container.h b/src/common/container.h index 125900c8ca..5abd8b48d9 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -109,9 +109,9 @@ void smartlist_sort_digests(smartlist_t *sl); void smartlist_sort_digests256(smartlist_t *sl); void smartlist_sort_pointers(smartlist_t *sl); -char *smartlist_get_most_frequent_string(smartlist_t *sl); -char *smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out); -char *smartlist_get_most_frequent_digest256(smartlist_t *sl); +const char *smartlist_get_most_frequent_string(smartlist_t *sl); +const char *smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out); +const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl); void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); @@ -361,7 +361,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, DECLARE_MAP_FNS(strmap_t, const char *, strmap_); /* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */ DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_); -/* Map from const uint8_t[DIGEST_LEN] to void *. Implemented with a hash +/* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash * table. */ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index bde800fd23..d54a5599cd 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -110,11 +110,11 @@ ## Use these to restrict the maximum traffic per day, week, or month. ## Note that this threshold applies separately to sent and received bytes, -## not to their sum: setting "4 GB" may allow up to 8 GB total before +## not to their sum: setting "40 GB" may allow up to 80 GB total before ## hibernating. ## -## Set a maximum of 4 gigabytes each way per period. -#AccountingMax 4 GBytes +## Set a maximum of 40 gigabytes each way per period. +#AccountingMax 40 GBytes ## Each period starts daily at midnight (AccountingMax is per day) #AccountingStart day 00:00 ## Each period starts on the 3rd of the month at 15:00 (AccountingMax diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index bde800fd23..d54a5599cd 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -110,11 +110,11 @@ ## Use these to restrict the maximum traffic per day, week, or month. ## Note that this threshold applies separately to sent and received bytes, -## not to their sum: setting "4 GB" may allow up to 8 GB total before +## not to their sum: setting "40 GB" may allow up to 80 GB total before ## hibernating. ## -## Set a maximum of 4 gigabytes each way per period. -#AccountingMax 4 GBytes +## Set a maximum of 40 gigabytes each way per period. +#AccountingMax 40 GBytes ## Each period starts daily at midnight (AccountingMax is per day) #AccountingStart day 00:00 ## Each period starts on the 3rd of the month at 15:00 (AccountingMax diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 0f3b77fe28..d8e6ee2229 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -478,7 +478,7 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, if (microdesc_digest256_out) { smartlist_t *digests = smartlist_new(); - const char *best_microdesc_digest; + const uint8_t *best_microdesc_digest; SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { char d[DIGEST256_LEN]; if (compare_vote_rs(rs, most)) diff --git a/src/or/main.c b/src/or/main.c index 5bff82b3cf..092014f7fa 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -982,19 +982,18 @@ conn_close_if_marked(int i) return 1; } -/** We've just tried every dirserver we know about, and none of - * them were reachable. Assume the network is down. Change state - * so next time an application connection arrives we'll delay it - * and try another directory fetch. Kill off all the circuit_wait - * streams that are waiting now, since they will all timeout anyway. +/** Implementation for directory_all_unreachable. This is done in a callback, + * since otherwise it would complicate Tor's control-flow graph beyond all + * reason. */ -void -directory_all_unreachable(time_t now) +static void +directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg) { - connection_t *conn; - (void)now; + (void)fd; + (void)event; + (void)arg; - stats_n_seconds_working=0; /* reset it */ + connection_t *conn; while ((conn = connection_get_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT))) { @@ -1010,6 +1009,31 @@ directory_all_unreachable(time_t now) control_event_general_status(LOG_ERR, "DIR_ALL_UNREACHABLE"); } +static struct event *directory_all_unreachable_cb_event = NULL; + +/** We've just tried every dirserver we know about, and none of + * them were reachable. Assume the network is down. Change state + * so next time an application connection arrives we'll delay it + * and try another directory fetch. Kill off all the circuit_wait + * streams that are waiting now, since they will all timeout anyway. + */ +void +directory_all_unreachable(time_t now) +{ + (void)now; + + stats_n_seconds_working=0; /* reset it */ + + if (!directory_all_unreachable_cb_event) { + directory_all_unreachable_cb_event = + tor_event_new(tor_libevent_get_base(), + -1, EV_READ, directory_all_unreachable_cb, NULL); + tor_assert(directory_all_unreachable_cb_event); + } + + event_active(directory_all_unreachable_cb_event, EV_READ, 1); +} + /** This function is called whenever we successfully pull down some new * network statuses or server descriptors. */ void @@ -1488,6 +1512,10 @@ run_scheduled_events(time_t now) #define CLEAN_CACHES_INTERVAL (30*60) time_to.clean_caches = now + CLEAN_CACHES_INTERVAL; } + /* We don't keep entries that are more than five minutes old so we try to + * clean it as soon as we can since we want to make sure the client waits + * as little as possible for reachability reasons. */ + rend_cache_failure_clean(now); #define RETRY_DNS_INTERVAL (10*60) /* If we're a server and initializing dns failed, retry periodically. */ @@ -1884,7 +1912,7 @@ ip_address_changed(int at_interface) if (at_interface) { if (! server) { /* Okay, change our keys. */ - if (init_keys()<0) + if (init_keys_client() < 0) log_warn(LD_GENERAL, "Unable to rotate keys after IP change!"); } } else { diff --git a/src/or/rendcache.c b/src/or/rendcache.c index be94ef4445..9a33046fb6 100644 --- a/src/or/rendcache.c +++ b/src/or/rendcache.c @@ -9,7 +9,6 @@ #include "rendcache.h" #include "config.h" -#include "rendcommon.h" #include "rephist.h" #include "routerlist.h" #include "routerparse.h" @@ -22,6 +21,33 @@ static strmap_t *rend_cache = NULL; * directories. */ static digestmap_t *rend_cache_v2_dir = NULL; +/** (Client side only) Map from service id to rend_cache_failure_t. This + * cache is used to track intro point(IP) failures so we know when to keep + * or discard a new descriptor we just fetched. Here is a description of the + * cache behavior. + * + * Everytime tor discards an IP (ex: receives a NACK), we add an entry to + * this cache noting the identity digest of the IP and it's failure type for + * the service ID. The reason we indexed this cache by service ID is to + * differentiate errors that can occur only for a specific service like a + * NACK for instance. It applies for one but maybe not for the others. + * + * Once a service descriptor is fetched and considered valid, each IP is + * looked up in this cache and if present, it is discarded from the fetched + * descriptor. At the end, all IP(s) in the cache, for a specific service + * ID, that were NOT present in the descriptor are removed from this cache. + * Which means that if at least one IP was not in this cache, thus usuable, + * it's considered a new descriptor so we keep it. Else, if all IPs were in + * this cache, we discard the descriptor as it's considered unsuable. + * + * Once a descriptor is removed from the rend cache or expires, the entry + * in this cache is also removed for the service ID. + * + * This scheme allows us to not realy on the descriptor's timestamp (which + * is rounded down to the hour) to know if we have a newer descriptor. We + * only rely on the usability of intro points from an internal state. */ +static strmap_t *rend_cache_failure = NULL; + /** DOCDOC */ static size_t rend_cache_total_allocation = 0; @@ -32,6 +58,7 @@ rend_cache_init(void) { rend_cache = strmap_new(); rend_cache_v2_dir = digestmap_new(); + rend_cache_failure = strmap_new(); } /** Return the approximate number of bytes needed to hold <b>e</b>. */ @@ -85,6 +112,83 @@ rend_cache_increment_allocation(size_t n) } } +/** Helper: free a rend cache failure intro object. */ +static void +rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t *entry) +{ + if (entry == NULL) { + return; + } + tor_free(entry); +} + +/** Allocate a rend cache failure intro object and return it. <b>failure</b> + * is set into the object. This function can not fail. */ +static rend_cache_failure_intro_t * +rend_cache_failure_intro_entry_new(rend_intro_point_failure_t failure) +{ + rend_cache_failure_intro_t *entry = tor_malloc(sizeof(*entry)); + entry->failure_type = failure; + entry->created_ts = time(NULL); + return entry; +} + +/** Helper: free a rend cache failure object. */ +static void +rend_cache_failure_entry_free(rend_cache_failure_t *entry) +{ + if (entry == NULL) { + return; + } + + /* Free and remove every intro failure object. */ + DIGESTMAP_FOREACH_MODIFY(entry->intro_failures, key, + rend_cache_failure_intro_t *, e) { + rend_cache_failure_intro_entry_free(e); + MAP_DEL_CURRENT(key); + } DIGESTMAP_FOREACH_END; + tor_free(entry); +} + +/** Helper: deallocate a rend_cache_failure_t. (Used with strmap_free(), + * which requires a function pointer whose argument is void*). */ +static void +rend_cache_failure_entry_free_(void *entry) +{ + rend_cache_failure_entry_free(entry); +} + +/** Allocate a rend cache failure object and return it. This function can + * not fail. */ +static rend_cache_failure_t * +rend_cache_failure_entry_new(void) +{ + rend_cache_failure_t *entry = tor_malloc(sizeof(*entry)); + entry->intro_failures = digestmap_new(); + return entry; +} + +/** Remove failure cache entry for the service ID in the given descriptor + * <b>desc</b>. */ +static void +rend_cache_failure_remove(rend_service_descriptor_t *desc) +{ + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + rend_cache_failure_t *entry; + + if (desc == NULL) { + return; + } + if (rend_get_service_id(desc->pk, service_id) < 0) { + return; + } + entry = strmap_get_lc(rend_cache_failure, service_id); + if (entry != NULL) { + strmap_remove_lc(rend_cache_failure, service_id); + rend_cache_failure_entry_free(entry); + } +} + /** Helper: free storage held by a single service descriptor cache entry. */ static void rend_cache_entry_free(rend_cache_entry_t *e) @@ -92,6 +196,9 @@ rend_cache_entry_free(rend_cache_entry_t *e) if (!e) return; rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); + /* We are about to remove a descriptor from the cache so remove the entry + * in the failure cache. */ + rend_cache_failure_remove(e->parsed); rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); @@ -111,11 +218,42 @@ rend_cache_free_all(void) { strmap_free(rend_cache, rend_cache_entry_free_); digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_); + strmap_free(rend_cache_failure, rend_cache_failure_entry_free_); rend_cache = NULL; rend_cache_v2_dir = NULL; + rend_cache_failure = NULL; rend_cache_total_allocation = 0; } +/** Remove all entries that re REND_CACHE_FAILURE_MAX_AGE old. This is + * called every second. + * + * We have to clean these regurlarly else if for whatever reasons an hidden + * service goes offline and a client tries to connect to it during that + * time, a failure entry is created and the client will be unable to connect + * for a while even though the service has return online. */ +void +rend_cache_failure_clean(time_t now) +{ + time_t cutoff = now - REND_CACHE_FAILURE_MAX_AGE; + STRMAP_FOREACH_MODIFY(rend_cache_failure, key, + rend_cache_failure_t *, ent) { + /* Free and remove every intro failure object that match the cutoff. */ + DIGESTMAP_FOREACH_MODIFY(ent->intro_failures, ip_key, + rend_cache_failure_intro_t *, ip_ent) { + if (ip_ent->created_ts < cutoff) { + rend_cache_failure_intro_entry_free(ip_ent); + MAP_DEL_CURRENT(ip_key); + } + } DIGESTMAP_FOREACH_END; + /* If the entry is now empty of intro point failures, remove it. */ + if (digestmap_isempty(ent->intro_failures)) { + rend_cache_failure_entry_free(ent); + MAP_DEL_CURRENT(key); + } + } STRMAP_FOREACH_END; +} + /** Removes all old entries from the service descriptor cache. */ void @@ -150,6 +288,140 @@ rend_cache_purge(void) rend_cache = strmap_new(); } +/** Remove ALL entries from the failure cache. This is also called when a + * NEWNYM signal is received. */ +void +rend_cache_failure_purge(void) +{ + if (rend_cache_failure) { + log_info(LD_REND, "Purging HS failure cache"); + strmap_free(rend_cache_failure, rend_cache_failure_entry_free_); + } + rend_cache_failure = strmap_new(); +} + +/** Lookup the rend failure cache using a relay identity digest in + * <b>identity</b> and service ID <b>service_id</b>. If found, the intro + * failure is set in <b>intro_entry</b> else it stays untouched. Return 1 + * iff found else 0. */ +static int +cache_failure_intro_lookup(const uint8_t *identity, const char *service_id, + rend_cache_failure_intro_t **intro_entry) +{ + rend_cache_failure_t *elem; + rend_cache_failure_intro_t *intro_elem; + + tor_assert(rend_cache_failure); + + if (intro_entry) { + *intro_entry = NULL; + } + + /* Lookup descriptor and return it. */ + elem = strmap_get_lc(rend_cache_failure, service_id); + if (elem == NULL) { + goto not_found; + } + intro_elem = digestmap_get(elem->intro_failures, (char *) identity); + if (intro_elem == NULL) { + goto not_found; + } + if (intro_entry) { + *intro_entry = intro_elem; + } + return 1; +not_found: + return 0; +} + +/** Add an intro point failure to the failure cache using the relay + * <b>identity</b> and service ID <b>service_id</b>. Record the + * <b>failure</b> in that object. */ +static void +cache_failure_intro_add(const uint8_t *identity, const char *service_id, + rend_intro_point_failure_t failure) +{ + rend_cache_failure_t *fail_entry; + rend_cache_failure_intro_t *entry; + + /* Make sure we have a failure object for this service ID and if not, + * create it with this new intro failure entry. */ + fail_entry = strmap_get_lc(rend_cache_failure, service_id); + if (fail_entry == NULL) { + fail_entry = rend_cache_failure_entry_new(); + /* Add failure entry to global rend failure cache. */ + strmap_set_lc(rend_cache_failure, service_id, fail_entry); + } + entry = rend_cache_failure_intro_entry_new(failure); + digestmap_set(fail_entry->intro_failures, (char *) identity, entry); +} + +/** Using a parsed descriptor <b>desc</b>, check if the introduction points + * are present in the failure cache and if so they are removed from the + * descriptor and kept into the failure cache. Then, each intro points that + * are NOT in the descriptor but in the failure cache for the given + * <b>service_id</b> are removed from the failure cache. */ +static void +validate_intro_point_failure(const rend_service_descriptor_t *desc, + const char *service_id) +{ + rend_cache_failure_t *new_entry, *cur_entry; + /* New entry for the service ID that will be replacing the one in the + * failure cache since we have a new descriptor. In the case where all + * intro points are removed, we are assured that the new entry is the same + * as the current one. */ + new_entry = tor_malloc(sizeof(*new_entry)); + new_entry->intro_failures = digestmap_new(); + + tor_assert(desc); + + SMARTLIST_FOREACH_BEGIN(desc->intro_nodes, rend_intro_point_t *, intro) { + int found; + rend_cache_failure_intro_t *entry; + const uint8_t *identity = + (uint8_t *) intro->extend_info->identity_digest; + + found = cache_failure_intro_lookup(identity, service_id, &entry); + if (found) { + /* This intro point is in our cache, discard it from the descriptor + * because chances are that it's unusable. */ + SMARTLIST_DEL_CURRENT(desc->intro_nodes, intro); + rend_intro_point_free(intro); + /* Keep it for our new entry. */ + digestmap_set(new_entry->intro_failures, (char *) identity, entry); + continue; + } + } SMARTLIST_FOREACH_END(intro); + + /* Swap the failure entry in the cache and free the current one. */ + cur_entry = strmap_get_lc(rend_cache_failure, service_id); + if (cur_entry != NULL) { + rend_cache_failure_entry_free(cur_entry); + } + strmap_set_lc(rend_cache_failure, service_id, new_entry); +} + +/** Note down an intro failure in the rend failure cache using the type of + * failure in <b>failure</b> for the relay identity digest in + * <b>identity</b> and service ID <b>service_id</b>. If an entry already + * exists in the cache, the failure type is changed with <b>failure</b>. */ +void +rend_cache_intro_failure_note(rend_intro_point_failure_t failure, + const uint8_t *identity, + const char *service_id) +{ + int found; + rend_cache_failure_intro_t *entry; + + found = cache_failure_intro_lookup(identity, service_id, &entry); + if (!found) { + cache_failure_intro_add(identity, service_id, failure); + } else { + /* Replace introduction point failure with this one. */ + entry->failure_type = failure; + } +} + /** Remove all old v2 descriptors and those for which this hidden service * directory is not responsible for any more. * @@ -537,20 +809,44 @@ rend_cache_store_v2_desc_as_client(const char *desc, "the future.", safe_str_client(service_id)); goto err; } - /* Do we already have a newer descriptor? */ + /* Do we have the same exact copy already in our cache? */ tor_snprintf(key, sizeof(key), "2%s", service_id); e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); - if (e && e->parsed->timestamp >= parsed->timestamp) { + if (e && !strcmp(desc, e->desc)) { + log_info(LD_REND,"We already have this service descriptor %s.", + safe_str_client(service_id)); + goto okay; + } + /* Verify that we are not replacing an older descriptor. It's important to + * avoid an evil HSDir serving old descriptor. We validate if the + * timestamp is greater than and not equal because it's a rounded down + * timestamp to the hour so if the descriptor changed in the same hour, + * the rend cache failure will tells us if we have a new descriptor. */ + if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a new enough service descriptor for " "service ID %s with the same desc ID and version.", safe_str_client(service_id)); goto okay; } + /* Lookup our failure cache for intro point that might be unsuable. */ + validate_intro_point_failure(parsed, service_id); + /* It's now possible that our intro point list is empty, this means that + * this descriptor is useless to us because intro points have all failed + * somehow before. Discard the descriptor. */ + if (smartlist_len(parsed->intro_nodes) == 0) { + log_info(LD_REND, "Service descriptor with service ID %s, every " + "intro points are unusable. Discarding it.", + safe_str_client(service_id)); + goto err; + } + /* Now either purge the current one and replace it's content or create a + * new one and add it to the rend cache. */ if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); } else { rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); + rend_cache_failure_remove(e->parsed); rend_service_descriptor_free(e->parsed); tor_free(e->desc); } diff --git a/src/or/rendcache.h b/src/or/rendcache.h index f61f02a8e6..0512058054 100644 --- a/src/or/rendcache.h +++ b/src/or/rendcache.h @@ -10,6 +10,7 @@ #define TOR_RENDCACHE_H #include "or.h" +#include "rendcommon.h" /** How old do we let hidden service descriptors get before discarding * them as too old? */ @@ -17,6 +18,8 @@ /** How wrong do we assume our clock may be when checking whether hidden * services are too old or too new? */ #define REND_CACHE_MAX_SKEW (24*60*60) +/** How old do we keep an intro point failure entry in the failure cache? */ +#define REND_CACHE_FAILURE_MAX_AGE (5*60) /* Do not allow more than this many introduction points in a hidden service * descriptor */ @@ -31,8 +34,23 @@ typedef struct rend_cache_entry_t { rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ } rend_cache_entry_t; +/* Introduction point failure type. */ +typedef struct rend_cache_failure_intro_t { + /* When this intro point failure occured thus we allocated this object and + * cache it. */ + time_t created_ts; + rend_intro_point_failure_t failure_type; +} rend_cache_failure_intro_t; + +/** Cache failure object indexed by service ID. */ +typedef struct rend_cache_failure_t { + /* Contains rend_cache_failure_intro_t indexed by identity digest. */ + digestmap_t *intro_failures; +} rend_cache_failure_t; + void rend_cache_init(void); void rend_cache_clean(time_t now); +void rend_cache_failure_clean(time_t now); void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove); void rend_cache_purge(void); void rend_cache_free_all(void); @@ -53,5 +71,10 @@ rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc, rend_cache_entry_t **entry); size_t rend_cache_get_total_allocation(void); +void rend_cache_intro_failure_note(rend_intro_point_failure_t failure, + const uint8_t *identity, + const char *service_id); +void rend_cache_failure_purge(void); + #endif /* TOR_RENDCACHE_H */ diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 59e938e89c..c6f29a7707 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -38,6 +38,7 @@ void rend_client_purge_state(void) { rend_cache_purge(); + rend_cache_failure_purge(); rend_client_cancel_descriptor_fetches(); rend_client_purge_last_hid_serv_requests(); } @@ -1019,6 +1020,9 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro, tor_fragile_assert(); /* fall through */ case INTRO_POINT_FAILURE_GENERIC: + rend_cache_intro_failure_note(failure_type, + (uint8_t *) failed_intro->identity_digest, + rend_query->onion_address); rend_intro_point_free(intro); smartlist_del(ent->parsed->intro_nodes, i); break; @@ -1034,6 +1038,9 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro, intro->unreachable_count, zap_intro_point ? " Removing from descriptor.": ""); if (zap_intro_point) { + rend_cache_intro_failure_note(failure_type, + (uint8_t *) failed_intro->identity_digest, + rend_query->onion_address); rend_intro_point_free(intro); smartlist_del(ent->parsed->intro_nodes, i); } diff --git a/src/or/rendclient.h b/src/or/rendclient.h index 439f42875b..124433ef31 100644 --- a/src/or/rendclient.h +++ b/src/or/rendclient.h @@ -26,10 +26,6 @@ int rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs); void rend_client_cancel_descriptor_fetches(void); void rend_client_purge_last_hid_serv_requests(void); -#define INTRO_POINT_FAILURE_GENERIC 0 -#define INTRO_POINT_FAILURE_TIMEOUT 1 -#define INTRO_POINT_FAILURE_UNREACHABLE 2 - int rend_client_report_intro_point_failure(extend_info_t *failed_intro, rend_data_t *rend_query, unsigned int failure_type); diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index ba5e077642..3b2f86d614 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -12,6 +12,12 @@ #ifndef TOR_RENDCOMMON_H #define TOR_RENDCOMMON_H +typedef enum rend_intro_point_failure_t { + INTRO_POINT_FAILURE_GENERIC = 0, + INTRO_POINT_FAILURE_TIMEOUT = 1, + INTRO_POINT_FAILURE_UNREACHABLE = 2, +} rend_intro_point_failure_t; + /** Free all storage associated with <b>data</b> */ static INLINE void rend_data_free(rend_data_t *data) diff --git a/src/or/router.c b/src/or/router.c index 47825e2d1c..03973ae90a 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -767,6 +767,46 @@ router_write_fingerprint(int hashed) return result; } +static int +init_keys_common(void) +{ + if (!key_lock) + key_lock = tor_mutex_new(); + + /* There are a couple of paths that put us here before we've asked + * openssl to initialize itself. */ + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { + log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); + return -1; + } + + return 0; +} + +int +init_keys_client(void) +{ + crypto_pk_t *prkey; + if (init_keys_common() < 0) + return -1; + + if (!(prkey = crypto_pk_new())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_pk_free(prkey); + return -1; + } + set_client_identity_key(prkey); + /* Create a TLS context. */ + if (router_initialize_tls_context() < 0) { + log_err(LD_GENERAL,"Error creating TLS context for Tor client."); + return -1; + } + return 0; +} + /** Initialize all OR private keys, and the TLS context, as necessary. * On OPs, this only initializes the tls context. Return 0 on success, * or -1 if Tor should die. @@ -786,35 +826,13 @@ init_keys(void) int v3_digest_set = 0; authority_cert_t *cert = NULL; - if (!key_lock) - key_lock = tor_mutex_new(); - - /* There are a couple of paths that put us here before we've asked - * openssl to initialize itself. */ - if (crypto_global_init(get_options()->HardwareAccel, - get_options()->AccelName, - get_options()->AccelDir)) { - log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); - return -1; - } - /* OP's don't need persistent keys; just make up an identity and * initialize the TLS context. */ if (!server_mode(options)) { - if (!(prkey = crypto_pk_new())) - return -1; - if (crypto_pk_generate_key(prkey)) { - crypto_pk_free(prkey); - return -1; - } - set_client_identity_key(prkey); - /* Create a TLS context. */ - if (router_initialize_tls_context() < 0) { - log_err(LD_GENERAL,"Error creating TLS context for Tor client."); - return -1; - } - return 0; + return init_keys_client(); } + if (init_keys_common() < 0) + return -1; /* Make sure DataDirectory exists, and is private. */ if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) { return -1; diff --git a/src/or/router.h b/src/or/router.h index 61b35d6b5a..d8fcf0a9ad 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -37,6 +37,7 @@ void ntor_key_map_free(di_digest256_map_t *map); int router_initialize_tls_context(void); int init_keys(void); +int init_keys_client(void); int check_whether_orport_reachable(void); int check_whether_dirport_reachable(void); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index dc48862201..aebbd480d2 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -3295,6 +3295,8 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, /* Make sure that it isn't expired. */ if (router->cert_expiration_time < approx_time()) { + routerinfo_free(router); + *msg = "Some certs on this router are expired."; return ROUTER_CERTS_EXPIRED; } diff --git a/src/test/test.h b/src/test/test.h index b0c0946ac4..86699c3d07 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -40,6 +40,15 @@ tt_assert_test_type(a,b,#a" "#op" "#b,double,(val1_ op val2_),"%g", \ TT_EXIT_TEST_FUNCTION) +/* Declare "double equal" in a sneaky way, so compiler won't complain about + * comparing floats with == or !=. Of course, only do this if you know what + * you're doing. */ +#define tt_double_eq(a,b) \ + STMT_BEGIN \ + tt_double_op((a), >=, (b)); \ + tt_double_op((a), <=, (b)); \ + STMT_END + #ifdef _MSC_VER #define U64_PRINTF_TYPE uint64_t #define I64_PRINTF_TYPE int64_t diff --git a/src/test/test_containers.c b/src/test/test_containers.c index 2ae81bf18d..1ee240fb0d 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -887,7 +887,7 @@ static void test_container_order_functions(void *arg) { int lst[25], n = 0; - unsigned int lst_2[25]; + uint32_t lst_2[25]; // int a=12,b=24,c=25,d=60,e=77; #define median() median_int(lst, n) @@ -933,6 +933,31 @@ test_container_order_functions(void *arg) #undef third_quartile + double dbls[] = { 1.0, 10.0, 100.0, 1e4, 1e5, 1e6 }; + tt_double_eq(1.0, median_double(dbls, 1)); + tt_double_eq(1.0, median_double(dbls, 2)); + tt_double_eq(10.0, median_double(dbls, 3)); + tt_double_eq(10.0, median_double(dbls, 4)); + tt_double_eq(100.0, median_double(dbls, 5)); + tt_double_eq(100.0, median_double(dbls, 6)); + + time_t times[] = { 5, 10, 20, 25, 15 }; + + tt_assert(5 == median_time(times, 1)); + tt_assert(5 == median_time(times, 2)); + tt_assert(10 == median_time(times, 3)); + tt_assert(10 == median_time(times, 4)); + tt_assert(15 == median_time(times, 5)); + + int32_t int32s[] = { -5, -10, -50, 100 }; + tt_int_op(-5, ==, median_int32(int32s, 1)); + tt_int_op(-10, ==, median_int32(int32s, 2)); + tt_int_op(-10, ==, median_int32(int32s, 3)); + tt_int_op(-10, ==, median_int32(int32s, 4)); + + long longs[] = { -30, 30, 100, -100, 7 }; + tt_int_op(7, ==, find_nth_long(longs, 5, 2)); + done: ; } @@ -1078,6 +1103,129 @@ test_container_fp_pair_map(void *arg) tor_free(v105); } +static void +test_container_smartlist_most_frequent(void *arg) +{ + (void) arg; + smartlist_t *sl = smartlist_new(); + + int count = -1; + const char *cp; + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 0); + tt_ptr_op(cp, ==, NULL); + + /* String must be sorted before we call get_most_frequent */ + smartlist_split_string(sl, "abc:def:ghi", ":", 0, 0); + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 1); + tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */ + + smartlist_split_string(sl, "def:ghi", ":", 0, 0); + smartlist_sort_strings(sl); + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 2); + tt_ptr_op(cp, !=, NULL); + tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */ + + smartlist_split_string(sl, "def:abc:qwop", ":", 0, 0); + smartlist_sort_strings(sl); + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 3); + tt_ptr_op(cp, !=, NULL); + tt_str_op(cp, ==, "def"); /* No tie */ + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); +} + +static void +test_container_smartlist_sort_ptrs(void *arg) +{ + (void)arg; + int array[10]; + int *arrayptrs[11]; + smartlist_t *sl = smartlist_new(); + unsigned i=0, j; + + for (j = 0; j < ARRAY_LENGTH(array); ++j) { + smartlist_add(sl, &array[j]); + arrayptrs[i++] = &array[j]; + if (j == 5) { + smartlist_add(sl, &array[j]); + arrayptrs[i++] = &array[j]; + } + } + + for (i = 0; i < 10; ++i) { + smartlist_shuffle(sl); + smartlist_sort_pointers(sl); + for (j = 0; j < ARRAY_LENGTH(arrayptrs); ++j) { + tt_ptr_op(smartlist_get(sl, j), ==, arrayptrs[j]); + } + } + + done: + smartlist_free(sl); +} + +static void +test_container_smartlist_strings_eq(void *arg) +{ + (void)arg; + smartlist_t *sl1 = smartlist_new(); + smartlist_t *sl2 = smartlist_new(); +#define EQ_SHOULD_SAY(s1,s2,val) \ + do { \ + SMARTLIST_FOREACH(sl1, char *, cp, tor_free(cp)); \ + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); \ + smartlist_clear(sl1); \ + smartlist_clear(sl2); \ + smartlist_split_string(sl1, (s1), ":", 0, 0); \ + smartlist_split_string(sl2, (s2), ":", 0, 0); \ + tt_int_op((val), OP_EQ, smartlist_strings_eq(sl1, sl2)); \ + } while (0) + + /* Both NULL, so equal */ + tt_int_op(1, ==, smartlist_strings_eq(NULL, NULL)); + + /* One NULL, not equal. */ + tt_int_op(0, ==, smartlist_strings_eq(NULL, sl1)); + tt_int_op(0, ==, smartlist_strings_eq(sl1, NULL)); + + /* Both empty, both equal. */ + EQ_SHOULD_SAY("", "", 1); + + /* One empty, not equal */ + EQ_SHOULD_SAY("", "ab", 0); + EQ_SHOULD_SAY("", "xy:z", 0); + EQ_SHOULD_SAY("abc", "", 0); + EQ_SHOULD_SAY("abc:cd", "", 0); + + /* Different lengths, not equal. */ + EQ_SHOULD_SAY("hello:world", "hello", 0); + EQ_SHOULD_SAY("hello", "hello:friends", 0); + + /* Same lengths, not equal */ + EQ_SHOULD_SAY("Hello:world", "goodbye:world", 0); + EQ_SHOULD_SAY("Hello:world", "Hello:stars", 0); + + /* Actually equal */ + EQ_SHOULD_SAY("ABC", "ABC", 1); + EQ_SHOULD_SAY(" ab : cd : e", " ab : cd : e", 1); + + done: + SMARTLIST_FOREACH(sl1, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl1); + smartlist_free(sl2); +} + #define CONTAINER_LEGACY(name) \ { #name, test_container_ ## name , 0, NULL, NULL } @@ -1099,6 +1247,9 @@ struct testcase_t container_tests[] = { CONTAINER_LEGACY(order_functions), CONTAINER(di_map, 0), CONTAINER_LEGACY(fp_pair_map), + CONTAINER(smartlist_most_frequent, 0), + CONTAINER(smartlist_sort_ptrs, 0), + CONTAINER(smartlist_strings_eq, 0), END_OF_TESTCASES }; diff --git a/src/test/test_util.c b/src/test/test_util.c index f8e766162d..a6c423dcdf 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4355,7 +4355,7 @@ test_util_get_avail_disk_space(void *arg) /* No answer for nonexistent directory */ val = tor_get_avail_disk_space("/akljasdfklsajdklasjkldjsa"); - tt_int_op(val, OP_EQ, -1); + tt_i64_op(val, OP_EQ, -1); /* Try the current directory */ val = tor_get_avail_disk_space("."); |