diff options
Diffstat (limited to 'src/or')
| -rw-r--r-- | src/or/hs_cache.c | 364 | ||||
| -rw-r--r-- | src/or/hs_cache.h | 53 | ||||
| -rw-r--r-- | src/or/hs_common.h | 2 | ||||
| -rw-r--r-- | src/or/hs_descriptor.c | 12 | ||||
| -rw-r--r-- | src/or/hs_descriptor.h | 2 | ||||
| -rw-r--r-- | src/or/include.am | 2 | ||||
| -rw-r--r-- | src/or/main.c | 3 | ||||
| -rw-r--r-- | src/or/relay.c | 5 | ||||
| -rw-r--r-- | src/or/rendcache.c | 63 | ||||
| -rw-r--r-- | src/or/rendcache.h | 13 |
10 files changed, 475 insertions, 44 deletions
diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c new file mode 100644 index 0000000000..c2e2914e5a --- /dev/null +++ b/src/or/hs_cache.c @@ -0,0 +1,364 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hs_cache.c + * \brief Handle hidden service descriptor caches. + **/ + +#include "hs_cache.h" + +#include "or.h" +#include "config.h" +#include "hs_common.h" +#include "hs_descriptor.h" +#include "rendcache.h" + +/* Directory descriptor cache. Map indexed by blinded key. */ +static digest256map_t *hs_cache_v3_dir; + +/* Remove a given descriptor from our cache. */ +static void +remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc) +{ + tor_assert(desc); + digest256map_remove(hs_cache_v3_dir, desc->key); +} + +/* Store a given descriptor in our cache. */ +static void +store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc) +{ + tor_assert(desc); + digest256map_set(hs_cache_v3_dir, desc->key, desc); +} + +/* Query our cache and return the entry or NULL if not found. */ +static hs_cache_dir_descriptor_t * +lookup_v3_desc_as_dir(const uint8_t *key) +{ + tor_assert(key); + return digest256map_get(hs_cache_v3_dir, key); +} + +/* Free a directory descriptor object. */ +static void +cache_dir_desc_free(hs_cache_dir_descriptor_t *desc) +{ + if (desc == NULL) { + return; + } + hs_desc_plaintext_data_free(desc->plaintext_data); + tor_free(desc->encoded_desc); + tor_free(desc); +} + +/* Create a new directory cache descriptor object from a encoded descriptor. + * On success, return the heap-allocated cache object, otherwise return NULL if + * we can't decode the descriptor. */ +static hs_cache_dir_descriptor_t * +cache_dir_desc_new(const char *desc) +{ + hs_cache_dir_descriptor_t *dir_desc; + + tor_assert(desc); + + dir_desc = tor_malloc_zero(sizeof(hs_cache_dir_descriptor_t)); + dir_desc->plaintext_data = + tor_malloc_zero(sizeof(hs_desc_plaintext_data_t)); + dir_desc->encoded_desc = tor_strdup(desc); + + if (hs_desc_decode_plaintext(desc, dir_desc->plaintext_data) < 0) { + log_debug(LD_DIR, "Unable to decode descriptor. Rejecting."); + goto err; + } + + /* The blinded pubkey is the indexed key. */ + dir_desc->key = dir_desc->plaintext_data->blinded_kp.pubkey.pubkey; + dir_desc->created_ts = time(NULL); + return dir_desc; + +err: + cache_dir_desc_free(dir_desc); + return NULL; +} + +/* Return the size of a cache entry in bytes. */ +static size_t +cache_get_entry_size(const hs_cache_dir_descriptor_t *entry) +{ + return (sizeof(*entry) + hs_desc_plaintext_obj_size(entry->plaintext_data) + + strlen(entry->encoded_desc)); +} + +/* Try to store a valid version 3 descriptor in the directory cache. Return 0 + * on success else a negative value is returned indicating that we have a + * newer version in our cache. On error, caller is responsible to free the + * given descriptor desc. */ +static int +cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc) +{ + hs_cache_dir_descriptor_t *cache_entry; + + tor_assert(desc); + + /* Verify if we have an entry in the cache for that key and if yes, check + * if we should replace it? */ + cache_entry = lookup_v3_desc_as_dir(desc->key); + if (cache_entry != NULL) { + /* Only replace descriptor if revision-counter is greater than the one + * in our cache */ + if (cache_entry->plaintext_data->revision_counter >= + desc->plaintext_data->revision_counter) { + log_info(LD_REND, "Descriptor revision counter in our cache is " + "greater or equal than the one we received. " + "Rejecting!"); + goto err; + } + /* We now know that the descriptor we just received is a new one so + * remove the entry we currently have from our cache so we can then + * store the new one. */ + remove_v3_desc_as_dir(cache_entry); + cache_dir_desc_free(cache_entry); + rend_cache_decrement_allocation(cache_get_entry_size(cache_entry)); + } + /* Store the descriptor we just got. We are sure here that either we + * don't have the entry or we have a newer descriptor and the old one + * has been removed from the cache. */ + store_v3_desc_as_dir(desc); + + /* Update our total cache size with this entry for the OOM. This uses the + * old HS protocol cache subsystem for which we are tied with. */ + rend_cache_increment_allocation(cache_get_entry_size(desc)); + + /* XXX: Update HS statistics. We should have specific stats for v3. */ + + return 0; + + err: + return -1; +} + +/* Using the query which is the blinded key for a descriptor version 3, lookup + * in our directory cache the entry. If found, 1 is returned and desc_out is + * populated with a newly allocated string being the encoded descriptor. If + * not found, 0 is returned and desc_out is untouched. On error, a negative + * value is returned and desc_out is untouched. */ +static int +cache_lookup_v3_as_dir(const char *query, char **desc_out) +{ + int found = 0; + ed25519_public_key_t blinded_key; + const hs_cache_dir_descriptor_t *entry; + + tor_assert(query); + + /* Decode blinded key using the given query value. */ + if (ed25519_public_from_base64(&blinded_key, query) < 0) { + log_info(LD_REND, "Unable to decode the v3 HSDir query %s.", + safe_str_client(query)); + goto err; + } + + entry = lookup_v3_desc_as_dir(blinded_key.pubkey); + if (entry != NULL) { + found = 1; + if (desc_out) { + *desc_out = tor_strdup(entry->encoded_desc); + } + } + + return found; + +err: + return -1; +} + +/* Clean the v3 cache by removing any entry that has expired using the + * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning + * process will use the lifetime found in the plaintext data section. Return + * the number of bytes cleaned. */ +static size_t +cache_clean_v3_as_dir(time_t now, time_t global_cutoff) +{ + size_t bytes_removed = 0; + + /* Code flow error if this ever happens. */ + tor_assert(global_cutoff >= 0); + + if (!hs_cache_v3_dir) { /* No cache to clean. Just return. */ + return 0; + } + + DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_dir, key, + hs_cache_dir_descriptor_t *, entry) { + size_t entry_size; + time_t cutoff = global_cutoff; + if (!cutoff) { + /* Cutoff is the lifetime of the entry found in the descriptor. */ + cutoff = now - entry->plaintext_data->lifetime_sec; + } + + /* If the entry has been created _after_ the cutoff, not expired so + * continue to the next entry in our v3 cache. */ + if (entry->created_ts > cutoff) { + continue; + } + /* Here, our entry has expired, remove and free. */ + MAP_DEL_CURRENT(key); + entry_size = cache_get_entry_size(entry); + bytes_removed += entry_size; + /* Entry is not in the cache anymore, destroy it. */ + cache_dir_desc_free(entry); + /* Update our cache entry allocation size for the OOM. */ + rend_cache_decrement_allocation(entry_size); + /* Logging. */ + { + char key_b64[BASE64_DIGEST256_LEN + 1]; + base64_encode(key_b64, sizeof(key_b64), (const char *) key, + DIGEST256_LEN, 0); + log_info(LD_REND, "Removing v3 descriptor '%s' from HSDir cache", + safe_str_client(key_b64)); + } + } DIGEST256MAP_FOREACH_END; + + return bytes_removed; +} + +/* Given an encoded descriptor, store it in the directory cache depending on + * which version it is. Return a negative value on error. On success, 0 is + * returned. */ +int +hs_cache_store_as_dir(const char *desc) +{ + hs_cache_dir_descriptor_t *dir_desc = NULL; + + tor_assert(desc); + + /* Create a new cache object. This can fail if the descriptor plaintext data + * is unparseable which in this case a log message will be triggered. */ + dir_desc = cache_dir_desc_new(desc); + if (dir_desc == NULL) { + goto err; + } + + /* Call the right function against the descriptor version. At this point, + * we are sure that the descriptor's version is supported else the + * decoding would have failed. */ + switch (dir_desc->plaintext_data->version) { + case 3: + default: + if (cache_store_v3_as_dir(dir_desc) < 0) { + goto err; + } + break; + } + return 0; + + err: + cache_dir_desc_free(dir_desc); + return -1; +} + +/* Using the query, lookup in our directory cache the entry. If found, 1 is + * returned and desc_out is populated with a newly allocated string being + * the encoded descriptor. If not found, 0 is returned and desc_out is + * untouched. On error, a negative value is returned and desc_out is + * untouched. */ +int +hs_cache_lookup_as_dir(uint32_t version, const char *query, + char **desc_out) +{ + int found; + + tor_assert(query); + /* This should never be called with an unsupported version. */ + tor_assert(hs_desc_is_supported_version(version)); + + switch (version) { + case 3: + default: + found = cache_lookup_v3_as_dir(query, desc_out); + break; + } + + return found; +} + +/* Clean all directory caches using the current time now. */ +void +hs_cache_clean_as_dir(time_t now) +{ + time_t cutoff; + + /* Start with v2 cache cleaning. */ + cutoff = now - rend_cache_max_entry_lifetime(); + rend_cache_clean_v2_descs_as_dir(cutoff); + + /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function + * to compute the cutoff by itself using the lifetime value. */ + cache_clean_v3_as_dir(now, 0); +} + +/* Do a round of OOM cleanup on all directory caches. Return the amount of + * removed bytes. It is possible that the returned value is lower than + * min_remove_bytes if the caches get emptied out so the caller should be + * aware of this. */ +size_t +hs_cache_handle_oom(time_t now, size_t min_remove_bytes) +{ + time_t k; + size_t bytes_removed = 0; + + /* Our OOM handler called with 0 bytes to remove is a code flow error. */ + tor_assert(min_remove_bytes != 0); + + /* The algorithm is as follow. K is the oldest expected descriptor age. + * + * 1) Deallocate all entries from v2 cache that are older than K hours. + * 1.1) If the amount of remove bytes has been reached, stop. + * 2) Deallocate all entries from v3 cache that are older than K hours + * 2.1) If the amount of remove bytes has been reached, stop. + * 3) Set K = K - RendPostPeriod and repeat process until K is < 0. + * + * This ends up being O(Kn). + */ + + /* Set K to the oldest expected age in seconds which is the maximum + * lifetime of a cache entry. We'll use the v2 lifetime because it's much + * bigger than the v3 thus leading to cleaning older descriptors. */ + k = rend_cache_max_entry_lifetime(); + + do { + time_t cutoff; + + /* If K becomes negative, it means we've empty the caches so stop and + * return what we were able to cleanup. */ + if (k < 0) { + break; + } + /* Compute a cutoff value with K and the current time. */ + cutoff = now - k; + + /* Start by cleaning the v2 cache with that cutoff. */ + bytes_removed += rend_cache_clean_v2_descs_as_dir(cutoff); + + if (bytes_removed < min_remove_bytes) { + /* We haven't remove enough bytes so clean v3 cache. */ + bytes_removed += cache_clean_v3_as_dir(now, cutoff); + /* Decrement K by a post period to shorten the cutoff. */ + k -= get_options()->RendPostPeriod; + } + } while (bytes_removed < min_remove_bytes); + + return bytes_removed; +} + +/* Initialize the hidden service cache subsystem. */ +void +hs_cache_init(void) +{ + /* Calling this twice is very wrong code flow. */ + tor_assert(!hs_cache_v3_dir); + hs_cache_v3_dir = digest256map_new(); +} diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h new file mode 100644 index 0000000000..3432d61b14 --- /dev/null +++ b/src/or/hs_cache.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hs_cache.h + * \brief Header file for hs_cache.c + **/ + +#ifndef TOR_HS_CACHE_H +#define TOR_HS_CACHE_H + +#include <stdint.h> + +#include "crypto.h" +#include "crypto_ed25519.h" +#include "hs_common.h" +#include "hs_descriptor.h" +#include "torcert.h" + +/* Descriptor representation on the directory side which is a subset of + * information that the HSDir can decode and serve it. */ +typedef struct hs_cache_dir_descriptor_t { + /* This object is indexed using the blinded pubkey located in the plaintext + * data which is populated only once the descriptor has been successfully + * decoded and validated. This simply points to that pubkey. */ + const uint8_t *key; + + /* When does this entry has been created. Used to expire entries. */ + time_t created_ts; + + /* Descriptor plaintext information. Obviously, we can't decrypt the + * encrypted part of the descriptor. */ + hs_desc_plaintext_data_t *plaintext_data; + + /* Encoded descriptor which is basically in text form. It's a NUL terminated + * string thus safe to strlen(). */ + char *encoded_desc; +} hs_cache_dir_descriptor_t; + +/* Public API */ + +void hs_cache_init(void); +void hs_cache_clean_as_dir(time_t now); +size_t hs_cache_handle_oom(time_t now, size_t min_remove_bytes); + +/* Store and Lookup function. They are version agnostic that is depending on + * the requested version of the descriptor, it will be re-routed to the + * right function. */ +int hs_cache_store_as_dir(const char *desc); +int hs_cache_lookup_as_dir(uint32_t version, const char *query, + char **desc_out); + +#endif /* TOR_HS_CACHE_H */ diff --git a/src/or/hs_common.h b/src/or/hs_common.h index e74fdc9f9e..a3edf618a7 100644 --- a/src/or/hs_common.h +++ b/src/or/hs_common.h @@ -3,7 +3,7 @@ /** * \file hs_common.h - * \brief Header file for hs_common.c. + * \brief Header file containing common data for the whole HS subsytem. **/ #ifndef TOR_HS_COMMON_H diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c index cdbea76605..9714d52f8c 100644 --- a/src/or/hs_descriptor.c +++ b/src/or/hs_descriptor.c @@ -1918,3 +1918,15 @@ hs_descriptor_free(hs_descriptor_t *desc) desc_encrypted_data_free_contents(&desc->encrypted_data); tor_free(desc); } + +/* Return the size in bytes of the given plaintext data object. A sizeof() is + * not enough because the object contains pointers and the encrypted blob. + * This is particularly useful for our OOM subsystem that tracks the HSDir + * cache size for instance. */ +size_t +hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data) +{ + tor_assert(data); + return (sizeof(*data) + sizeof(*data->signing_key_cert) + + data->encrypted_blob_size); +} diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h index 9b48096d0e..1213597984 100644 --- a/src/or/hs_descriptor.h +++ b/src/or/hs_descriptor.h @@ -207,6 +207,8 @@ int hs_desc_decode_plaintext(const char *encoded, int hs_desc_decode_encrypted(const hs_descriptor_t *desc, hs_desc_encrypted_data_t *desc_out); +size_t hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data); + #ifdef HS_DESCRIPTOR_PRIVATE /* Encoding. */ diff --git a/src/or/include.am b/src/or/include.am index e28bda41d9..02d67fa192 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -48,6 +48,7 @@ LIBTOR_A_SOURCES = \ src/or/entrynodes.c \ src/or/ext_orport.c \ src/or/hibernate.c \ + src/or/hs_cache.c \ src/or/hs_common.c \ src/or/hs_descriptor.c \ src/or/keypin.c \ @@ -159,6 +160,7 @@ ORHEADERS = \ src/or/geoip.h \ src/or/entrynodes.h \ src/or/hibernate.h \ + src/or/hs_cache.h \ src/or/hs_common.h \ src/or/hs_descriptor.h \ src/or/keypin.h \ diff --git a/src/or/main.c b/src/or/main.c index 03c2b7ed58..9defdf051a 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -37,6 +37,7 @@ #include "entrynodes.h" #include "geoip.h" #include "hibernate.h" +#include "hs_cache.h" #include "keypin.h" #include "main.h" #include "microdesc.h" @@ -1651,7 +1652,7 @@ clean_caches_callback(time_t now, const or_options_t *options) rep_history_clean(now - options->RephistTrackTime); rend_cache_clean(now, REND_CACHE_TYPE_CLIENT); rend_cache_clean(now, REND_CACHE_TYPE_SERVICE); - rend_cache_clean_v2_descs_as_dir(now, 0); + hs_cache_clean_as_dir(now); microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) return CLEAN_CACHES_INTERVAL; diff --git a/src/or/relay.c b/src/or/relay.c index 38096ad1bb..e2963e3288 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -25,6 +25,7 @@ #include "connection_or.h" #include "control.h" #include "geoip.h" +#include "hs_cache.h" #include "main.h" #include "networkstatus.h" #include "nodelist.h" @@ -2404,9 +2405,7 @@ cell_queues_check_size(void) if (rend_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = rend_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - rend_cache_clean_v2_descs_as_dir(time(NULL), bytes_to_remove); - alloc -= rend_cache_total; - alloc += rend_cache_get_total_allocation(); + alloc -= hs_cache_handle_oom(time(NULL), bytes_to_remove); } circuits_handle_oom(alloc); return 1; diff --git a/src/or/rendcache.c b/src/or/rendcache.c index 709aaffdee..bf43407289 100644 --- a/src/or/rendcache.c +++ b/src/or/rendcache.c @@ -86,7 +86,7 @@ rend_cache_get_total_allocation(void) } /** Decrement the total bytes attributed to the rendezvous cache by n. */ -STATIC void +void rend_cache_decrement_allocation(size_t n) { static int have_underflowed = 0; @@ -103,7 +103,7 @@ rend_cache_decrement_allocation(size_t n) } /** Increase the total bytes attributed to the rendezvous cache by n. */ -STATIC void +void rend_cache_increment_allocation(size_t n) { static int have_overflowed = 0; @@ -462,45 +462,36 @@ rend_cache_intro_failure_note(rend_intro_point_failure_t failure, } /** Remove all old v2 descriptors and those for which this hidden service - * directory is not responsible for any more. - * - * If at all possible, remove at least <b>force_remove</b> bytes of data. - */ -void -rend_cache_clean_v2_descs_as_dir(time_t now, size_t force_remove) + * directory is not responsible for any more. The cutoff is the time limit for + * which we want to keep the cache entry. In other words, any entry created + * before will be removed. */ +size_t +rend_cache_clean_v2_descs_as_dir(time_t cutoff) { digestmap_iter_t *iter; - time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW; - const int LAST_SERVED_CUTOFF_STEP = 1800; - time_t last_served_cutoff = cutoff; size_t bytes_removed = 0; - do { - for (iter = digestmap_iter_init(rend_cache_v2_dir); - !digestmap_iter_done(iter); ) { - const char *key; - void *val; - rend_cache_entry_t *ent; - digestmap_iter_get(iter, &key, &val); - ent = val; - if (ent->parsed->timestamp < cutoff || - ent->last_served < last_served_cutoff) { - char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; - base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); - log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str_client(key_base32)); - bytes_removed += rend_cache_entry_allocation(ent); - iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - rend_cache_entry_free(ent); - } else { - iter = digestmap_iter_next(rend_cache_v2_dir, iter); - } + + for (iter = digestmap_iter_init(rend_cache_v2_dir); + !digestmap_iter_done(iter); ) { + const char *key; + void *val; + rend_cache_entry_t *ent; + digestmap_iter_get(iter, &key, &val); + ent = val; + if (ent->parsed->timestamp < cutoff) { + char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); + log_info(LD_REND, "Removing descriptor with ID '%s' from cache", + safe_str_client(key_base32)); + bytes_removed += rend_cache_entry_allocation(ent); + iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); + rend_cache_entry_free(ent); + } else { + iter = digestmap_iter_next(rend_cache_v2_dir, iter); } + } - /* In case we didn't remove enough bytes, advance the cutoff a little. */ - last_served_cutoff += LAST_SERVED_CUTOFF_STEP; - if (last_served_cutoff > now) - break; - } while (bytes_removed < force_remove); + return bytes_removed; } /** Lookup in the client cache the given service ID <b>query</b> for diff --git a/src/or/rendcache.h b/src/or/rendcache.h index 270b614c38..746f142fcc 100644 --- a/src/or/rendcache.h +++ b/src/or/rendcache.h @@ -53,10 +53,17 @@ typedef enum { REND_CACHE_TYPE_SERVICE = 2, } rend_cache_type_t; +/* Return maximum lifetime in seconds of a cache entry. */ +static inline time_t +rend_cache_max_entry_lifetime(void) +{ + return REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW; +} + void rend_cache_init(void); void rend_cache_clean(time_t now, rend_cache_type_t cache_type); void rend_cache_failure_clean(time_t now); -void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove); +size_t rend_cache_clean_v2_descs_as_dir(time_t cutoff); void rend_cache_purge(void); void rend_cache_free_all(void); int rend_cache_lookup_entry(const char *query, int version, @@ -77,6 +84,8 @@ void rend_cache_intro_failure_note(rend_intro_point_failure_t failure, const uint8_t *identity, const char *service_id); void rend_cache_failure_purge(void); +void rend_cache_decrement_allocation(size_t n); +void rend_cache_increment_allocation(size_t n); #ifdef RENDCACHE_PRIVATE @@ -89,8 +98,6 @@ STATIC int cache_failure_intro_lookup(const uint8_t *identity, const char *service_id, rend_cache_failure_intro_t **intro_entry); -STATIC void rend_cache_decrement_allocation(size_t n); -STATIC void rend_cache_increment_allocation(size_t n); STATIC rend_cache_failure_intro_t *rend_cache_failure_intro_entry_new( rend_intro_point_failure_t failure); STATIC rend_cache_failure_t *rend_cache_failure_entry_new(void); |