Move literally everything out of src/or

This commit won't build yet -- it just puts everything in a slightly
more logical place.

The reasoning here is that "src/core" will hold the stuff that every (or
nearly every) tor instance will need in order to do onion routing.
Other features (including some necessary ones) will live in
"src/feature".  The "src/app" directory will hold the stuff needed
to have Tor be an application you can actually run.

This commit DOES NOT refactor the former contents of src/or into a
logical set of acyclic libraries, or change any code at all.  That
will have to come in the future.

We will continue to move things around and split them in the future,
but I hope this lays a reasonable groundwork for doing so.

30 files changed, 21696 insertions, 0 deletions

diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h
new file mode 100644
index 0000000000..19c3fda2de
--- /dev/null
+++ b/src/feature/nodelist/authority_cert_st.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef AUTHORITY_CERT_ST_H
+#define AUTHORITY_CERT_ST_H
+
+#include "or/signed_descriptor_st.h"
+
+/** Certificate for v3 directory protocol: binds long-term authority identity
+ * keys to medium-term authority signing keys. */
+struct authority_cert_t {
+  /** Information relating to caching this cert on disk and looking it up. */
+  signed_descriptor_t cache_info;
+  /** This authority's long-term authority identity key. */
+  crypto_pk_t *identity_key;
+  /** This authority's medium-term signing key. */
+  crypto_pk_t *signing_key;
+  /** The digest of <b>signing_key</b> */
+  char signing_key_digest[DIGEST_LEN];
+  /** The listed expiration time of this certificate. */
+  time_t expires;
+  /** This authority's IPv4 address, in host order. */
+  uint32_t addr;
+  /** This authority's directory port. */
+  uint16_t dir_port;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h
new file mode 100644
index 0000000000..168a83b230
--- /dev/null
+++ b/src/feature/nodelist/desc_store_st.h
@@ -0,0 +1,39 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DESC_STORE_ST_H
+#define DESC_STORE_ST_H
+
+/** Allowable types of desc_store_t. */
+typedef enum store_type_t {
+  ROUTER_STORE = 0,
+  EXTRAINFO_STORE = 1
+} store_type_t;
+
+/** A 'store' is a set of descriptors saved on disk, with accompanying
+ * journal, mmaped as needed, rebuilt as needed. */
+struct desc_store_t {
+  /** Filename (within DataDir) for the store.  We append .tmp to this
+   * filename for a temporary file when rebuilding the store, and .new to this
+   * filename for the journal. */
+  const char *fname_base;
+  /** Human-readable description of what this store contains. */
+  const char *description;
+
+  tor_mmap_t *mmap; /**< A mmap for the main file in the store. */
+
+  store_type_t type; /**< What's stored in this store? */
+
+  /** The size of the router log, in bytes. */
+  size_t journal_len;
+  /** The size of the router store, in bytes. */
+  size_t store_len;
+  /** Total bytes dropped since last rebuild: this is space currently
+   * used in the cache and the journal that could be freed by a rebuild. */
+  size_t bytes_dropped;
+};
+
+#endif
diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h
new file mode 100644
index 0000000000..0291e099bf
--- /dev/null
+++ b/src/feature/nodelist/document_signature_st.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DOCUMENT_SIGNATURE_ST_H
+#define DOCUMENT_SIGNATURE_ST_H
+
+/** A signature of some document by an authority. */
+struct document_signature_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
+  /** Declared SHA-1 digest of signing key used by this voter. */
+  char signing_key_digest[DIGEST_LEN];
+  /** Algorithm used to compute the digest of the document. */
+  digest_algorithm_t alg;
+  /** Signature of the signed thing. */
+  char *signature;
+  /** Length of <b>signature</b> */
+  int signature_len;
+  unsigned int bad_signature : 1; /**< Set to true if we've tried to verify
+                                   * the sig, and we know it's bad. */
+  unsigned int good_signature : 1; /**< Set to true if we've verified the sig
+                                     * as good. */
+};
+
+#endif
+
diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h
new file mode 100644
index 0000000000..f91bba7b68
--- /dev/null
+++ b/src/feature/nodelist/extrainfo_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef EXTRAINFO_ST_H
+#define EXTRAINFO_ST_H
+
+#include "or/signed_descriptor_st.h"
+
+/** Information needed to keep and cache a signed extra-info document. */
+struct extrainfo_t {
+  signed_descriptor_t cache_info;
+  /** SHA256 digest of this document */
+  uint8_t digest256[DIGEST256_LEN];
+  /** The router's nickname. */
+  char nickname[MAX_NICKNAME_LEN+1];
+  /** True iff we found the right key for this extra-info, verified the
+   * signature, and found it to be bad. */
+  unsigned int bad_sig : 1;
+  /** If present, we didn't have the right key to verify this extra-info,
+   * so this is a copy of the signature in the document. */
+  char *pending_sig;
+  /** Length of pending_sig. */
+  size_t pending_sig_len;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
new file mode 100644
index 0000000000..95c5e8b6f7
--- /dev/null
+++ b/src/feature/nodelist/microdesc.c
@@ -0,0 +1,1059 @@
+/* Copyright (c) 2009-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file microdesc.c
+ *
+ * \brief Implements microdescriptors -- an abbreviated description of
+ *  less-frequently-changing router information.
+ */
+
+#include "or/or.h"
+
+#include "lib/fdio/fdio.h"
+
+#include "or/circuitbuild.h"
+#include "or/config.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+
+#include "or/microdesc_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/routerstatus_st.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+/** A data structure to hold a bunch of cached microdescriptors.  There are
+ * two active files in the cache: a "cache file" that we mmap, and a "journal
+ * file" that we append to.  Periodically, we rebuild the cache file to hold
+ * only the microdescriptors that we want to keep */
+struct microdesc_cache_t {
+  /** Map from sha256-digest to microdesc_t for every microdesc_t in the
+   * cache. */
+  HT_HEAD(microdesc_map, microdesc_t) map;
+
+  /** Name of the cache file. */
+  char *cache_fname;
+  /** Name of the journal file. */
+  char *journal_fname;
+  /** Mmap'd contents of the cache file, or NULL if there is none. */
+  tor_mmap_t *cache_content;
+  /** Number of bytes used in the journal file. */
+  size_t journal_len;
+  /** Number of bytes in descriptors removed as too old. */
+  size_t bytes_dropped;
+
+  /** Total bytes of microdescriptor bodies we have added to this cache */
+  uint64_t total_len_seen;
+  /** Total number of microdescriptors we have added to this cache */
+  unsigned n_seen;
+
+  /** True iff we have loaded this cache from disk ever. */
+  int is_loaded;
+};
+
+static microdesc_cache_t *get_microdesc_cache_noload(void);
+
+/** Helper: computes a hash of <b>md</b> to place it in a hash table. */
+static inline unsigned int
+microdesc_hash_(microdesc_t *md)
+{
+  return (unsigned) siphash24g(md->digest, sizeof(md->digest));
+}
+
+/** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
+ * purposes. */
+static inline int
+microdesc_eq_(microdesc_t *a, microdesc_t *b)
+{
+  return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(microdesc_map, microdesc_t, node,
+             microdesc_hash_, microdesc_eq_)
+HT_GENERATE2(microdesc_map, microdesc_t, node,
+             microdesc_hash_, microdesc_eq_, 0.6,
+             tor_reallocarray_, tor_free_)
+
+/************************* md fetch fail cache *****************************/
+
+/* If we end up with too many outdated dirservers, something probably went
+ * wrong so clean up the list. */
+#define TOO_MANY_OUTDATED_DIRSERVERS 30
+
+/** List of dirservers with outdated microdesc information. The smartlist is
+ *  filled with the hex digests of outdated dirservers. */
+static smartlist_t *outdated_dirserver_list = NULL;
+
+/** Note that we failed to fetch a microdescriptor from the relay with
+ *  <b>relay_digest</b> (of size DIGEST_LEN). */
+void
+microdesc_note_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  /* Don't register outdated dirservers if we don't have a live consensus,
+   * since we might be trying to fetch microdescriptors that are not even
+   * currently active. */
+  if (!networkstatus_get_live_consensus(approx_time())) {
+    return;
+  }
+
+  if (!outdated_dirserver_list) {
+    outdated_dirserver_list = smartlist_new();
+  }
+
+  tor_assert(outdated_dirserver_list);
+
+  /* If the list grows too big, clean it up */
+  if (BUG(smartlist_len(outdated_dirserver_list) >
+          TOO_MANY_OUTDATED_DIRSERVERS)) {
+    microdesc_reset_outdated_dirservers_list();
+  }
+
+  /* Turn the binary relay digest to a hex since smartlists have better support
+   * for strings than digests. */
+  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Make sure we don't add a dirauth as an outdated dirserver */
+  if (router_get_trusteddirserver_by_digest(relay_digest)) {
+    log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
+    return;
+  }
+
+  /* Don't double-add outdated dirservers */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return;
+  }
+
+  /* Add it to the list of outdated dirservers */
+  smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
+
+  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
+}
+
+/** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
+ *  outdated dirserver */
+int
+microdesc_relay_is_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  if (!outdated_dirserver_list) {
+    return 0;
+  }
+
+  /* Convert identity digest to hex digest */
+  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Last time we tried to fetch microdescs, was this directory mirror missing
+   * any mds we asked for? */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Reset the list of outdated dirservers. */
+void
+microdesc_reset_outdated_dirservers_list(void)
+{
+  if (!outdated_dirserver_list) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+  smartlist_clear(outdated_dirserver_list);
+}
+
+/****************************************************************************/
+
+/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
+ * On success, return the total number of bytes written, and set
+ * *<b>annotation_len_out</b> to the number of bytes written as
+ * annotations. */
+static ssize_t
+dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
+{
+  ssize_t r = 0;
+  ssize_t written;
+  if (md->body == NULL) {
+    *annotation_len_out = 0;
+    return 0;
+  }
+  /* XXXX drops unknown annotations. */
+  if (md->last_listed) {
+    char buf[ISO_TIME_LEN+1];
+    char annotation[ISO_TIME_LEN+32];
+    format_iso_time(buf, md->last_listed);
+    tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
+    if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
+      log_warn(LD_DIR,
+               "Couldn't write microdescriptor annotation: %s",
+               strerror(errno));
+      return -1;
+    }
+    r += strlen(annotation);
+    *annotation_len_out = r;
+  } else {
+    *annotation_len_out = 0;
+  }
+
+  md->off = tor_fd_getpos(fd);
+  written = write_all_to_fd(fd, md->body, md->bodylen);
+  if (written != (ssize_t)md->bodylen) {
+    written = written < 0 ? 0 : written;
+    log_warn(LD_DIR,
+             "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
+             (long)written, (unsigned long)md->bodylen,
+             strerror(errno));
+    return -1;
+  }
+  r += md->bodylen;
+  return r;
+}
+
+/** Holds a pointer to the current microdesc_cache_t object, or NULL if no
+ * such object has been allocated. */
+static microdesc_cache_t *the_microdesc_cache = NULL;
+
+/** Return a pointer to the microdescriptor cache, loading it if necessary. */
+microdesc_cache_t *
+get_microdesc_cache(void)
+{
+  microdesc_cache_t *cache = get_microdesc_cache_noload();
+  if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
+    microdesc_cache_reload(cache);
+  }
+  return cache;
+}
+
+/** Return a pointer to the microdescriptor cache, creating (but not loading)
+ * it if necessary. */
+static microdesc_cache_t *
+get_microdesc_cache_noload(void)
+{
+  if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
+    microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
+    HT_INIT(microdesc_map, &cache->map);
+    cache->cache_fname = get_cachedir_fname("cached-microdescs");
+    cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
+    the_microdesc_cache = cache;
+  }
+  return the_microdesc_cache;
+}
+
+/* There are three sources of microdescriptors:
+   1) Generated by us while acting as a directory authority.
+   2) Loaded from the cache on disk.
+   3) Downloaded.
+*/
+
+/** Decode the microdescriptors from the string starting at <b>s</b> and
+ * ending at <b>eos</b>, and store them in <b>cache</b>.  If <b>no_save</b>,
+ * mark them as non-writable to disk.  If <b>where</b> is SAVED_IN_CACHE,
+ * leave their bodies as pointers to the mmap'd cache.  If where is
+ * <b>SAVED_NOWHERE</b>, do not allow annotations.  If listed_at is not -1,
+ * set the last_listed field of every microdesc to listed_at.  If
+ * requested_digests is non-null, then it contains a list of digests we mean
+ * to allow, so we should reject any non-requested microdesc with a different
+ * digest, and alter the list to contain only the digests of those microdescs
+ * we didn't find.
+ * Return a newly allocated list of the added microdescriptors, or NULL  */
+smartlist_t *
+microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save, time_t listed_at,
+                        smartlist_t *requested_digests256)
+{
+  void * const DIGEST_REQUESTED = (void*)1;
+  void * const DIGEST_RECEIVED = (void*)2;
+  void * const DIGEST_INVALID = (void*)3;
+
+  smartlist_t *descriptors, *added;
+  const int allow_annotations = (where != SAVED_NOWHERE);
+  smartlist_t *invalid_digests = smartlist_new();
+
+  descriptors = microdescs_parse_from_string(s, eos,
+                                             allow_annotations,
+                                             where, invalid_digests);
+  if (listed_at != (time_t)-1) {
+    SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
+                      md->last_listed = listed_at);
+  }
+  if (requested_digests256) {
+    digest256map_t *requested;
+    requested = digest256map_new();
+    /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
+    SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
+      digest256map_set(requested, cp, DIGEST_REQUESTED));
+    /* Set requested[d] to DIGEST_INVALID for every md we requested which we
+     * will never be able to parse.  Remove the ones we didn't request from
+     * invalid_digests.
+     */
+    SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
+      if (digest256map_get(requested, cp)) {
+        digest256map_set(requested, cp, DIGEST_INVALID);
+      } else {
+        tor_free(cp);
+        SMARTLIST_DEL_CURRENT(invalid_digests, cp);
+      }
+    } SMARTLIST_FOREACH_END(cp);
+    /* Update requested[d] to 2 for the mds we asked for and got. Delete the
+     * ones we never requested from the 'descriptors' smartlist.
+     */
+    SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+      if (digest256map_get(requested, (const uint8_t*)md->digest)) {
+        digest256map_set(requested, (const uint8_t*)md->digest,
+                         DIGEST_RECEIVED);
+      } else {
+        log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
+        microdesc_free(md);
+        SMARTLIST_DEL_CURRENT(descriptors, md);
+      }
+    } SMARTLIST_FOREACH_END(md);
+    /* Remove the ones we got or the invalid ones from requested_digests256.
+     */
+    SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
+      void *status = digest256map_get(requested, cp);
+      if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
+        tor_free(cp);
+        SMARTLIST_DEL_CURRENT(requested_digests256, cp);
+      }
+    } SMARTLIST_FOREACH_END(cp);
+    digest256map_free(requested, NULL);
+  }
+
+  /* For every requested microdescriptor that was unparseable, mark it
+   * as not to be retried. */
+  if (smartlist_len(invalid_digests)) {
+    networkstatus_t *ns =
+      networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+    if (ns) {
+      SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
+        routerstatus_t *rs =
+          router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
+        if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
+          download_status_mark_impossible(&rs->dl_status);
+        }
+      } SMARTLIST_FOREACH_END(d);
+    }
+  }
+  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
+  smartlist_free(invalid_digests);
+
+  added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
+  smartlist_free(descriptors);
+  return added;
+}
+
+/** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
+ * a string to decode.  Frees any members of <b>descriptors</b> that it does
+ * not add. */
+smartlist_t *
+microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                             smartlist_t *descriptors, saved_location_t where,
+                             int no_save)
+{
+  smartlist_t *added;
+  open_file_t *open_file = NULL;
+  int fd = -1;
+  //  int n_added = 0;
+  ssize_t size = 0;
+
+  if (where == SAVED_NOWHERE && !no_save) {
+    fd = start_writing_to_file(cache->journal_fname,
+                               OPEN_FLAGS_APPEND|O_BINARY,
+                               0600, &open_file);
+    if (fd < 0) {
+      log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
+               cache->journal_fname, strerror(errno));
+    }
+  }
+
+  added = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+    microdesc_t *md2;
+    md2 = HT_FIND(microdesc_map, &cache->map, md);
+    if (md2) {
+      /* We already had this one. */
+      if (md2->last_listed < md->last_listed)
+        md2->last_listed = md->last_listed;
+      microdesc_free(md);
+      if (where != SAVED_NOWHERE)
+        cache->bytes_dropped += size;
+      continue;
+    }
+
+    /* Okay, it's a new one. */
+    if (fd >= 0) {
+      size_t annotation_len;
+      size = dump_microdescriptor(fd, md, &annotation_len);
+      if (size < 0) {
+        /* we already warned in dump_microdescriptor */
+        abort_writing_to_file(open_file);
+        fd = -1;
+      } else {
+        md->saved_location = SAVED_IN_JOURNAL;
+        cache->journal_len += size;
+      }
+    } else {
+      md->saved_location = where;
+    }
+
+    md->no_save = no_save;
+
+    HT_INSERT(microdesc_map, &cache->map, md);
+    md->held_in_map = 1;
+    smartlist_add(added, md);
+    ++cache->n_seen;
+    cache->total_len_seen += md->bodylen;
+  } SMARTLIST_FOREACH_END(md);
+
+  if (fd >= 0) {
+    if (finish_writing_to_file(open_file) < 0) {
+      log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
+               strerror(errno));
+      smartlist_clear(added);
+      return added;
+    }
+  }
+
+  {
+    networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns && ns->flavor == FLAV_MICRODESC)
+      SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
+  }
+
+  if (smartlist_len(added))
+    router_dir_info_changed();
+
+  return added;
+}
+
+/** Remove every microdescriptor in <b>cache</b>. */
+void
+microdesc_cache_clear(microdesc_cache_t *cache)
+{
+  microdesc_t **entry, **next;
+
+  for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
+    microdesc_t *md = *entry;
+    next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
+    md->held_in_map = 0;
+    microdesc_free(md);
+  }
+  HT_CLEAR(microdesc_map, &cache->map);
+  if (cache->cache_content) {
+    int res = tor_munmap_file(cache->cache_content);
+    if (res != 0) {
+      log_warn(LD_FS,
+               "tor_munmap_file() failed clearing microdesc cache; "
+               "we are probably about to leak memory.");
+      /* TODO something smarter? */
+    }
+    cache->cache_content = NULL;
+  }
+  cache->total_len_seen = 0;
+  cache->n_seen = 0;
+  cache->bytes_dropped = 0;
+}
+
+/** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
+ * for the first time.  Return 0 on success, -1 on failure. */
+int
+microdesc_cache_reload(microdesc_cache_t *cache)
+{
+  struct stat st;
+  char *journal_content;
+  smartlist_t *added;
+  tor_mmap_t *mm;
+  int total = 0;
+
+  microdesc_cache_clear(cache);
+
+  cache->is_loaded = 1;
+
+  mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
+  if (mm) {
+    added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
+                                    SAVED_IN_CACHE, 0, -1, NULL);
+    if (added) {
+      total += smartlist_len(added);
+      smartlist_free(added);
+    }
+  }
+
+  journal_content = read_file_to_str(cache->journal_fname,
+                                     RFTS_IGNORE_MISSING, &st);
+  if (journal_content) {
+    cache->journal_len = (size_t) st.st_size;
+    added = microdescs_add_to_cache(cache, journal_content,
+                                    journal_content+st.st_size,
+                                    SAVED_IN_JOURNAL, 0, -1, NULL);
+    if (added) {
+      total += smartlist_len(added);
+      smartlist_free(added);
+    }
+    tor_free(journal_content);
+  }
+  log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
+           total);
+
+  microdesc_cache_rebuild(cache, 0 /* don't force */);
+
+  return 0;
+}
+
+/** By default, we remove any microdescriptors that have gone at least this
+ * long without appearing in a current consensus. */
+#define TOLERATE_MICRODESC_AGE (7*24*60*60)
+
+/** Remove all microdescriptors from <b>cache</b> that haven't been listed for
+ * a long time.  Does not rebuild the cache on disk.  If <b>cutoff</b> is
+ * positive, specifically remove microdescriptors that have been unlisted
+ * since <b>cutoff</b>.  If <b>force</b> is true, remove microdescriptors even
+ * if we have no current live microdescriptor consensus.
+ */
+void
+microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
+{
+  microdesc_t **mdp, *victim;
+  int dropped=0, kept=0;
+  size_t bytes_dropped = 0;
+  time_t now = time(NULL);
+
+  /* If we don't know a live consensus, don't believe last_listed values: we
+   * might be starting up after being down for a while. */
+  if (! force &&
+      ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
+      return;
+
+  if (cutoff <= 0)
+    cutoff = now - TOLERATE_MICRODESC_AGE;
+
+  for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
+    const int is_old = (*mdp)->last_listed < cutoff;
+    const unsigned held_by_nodes = (*mdp)->held_by_nodes;
+    if (is_old && !held_by_nodes) {
+      ++dropped;
+      victim = *mdp;
+      mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
+      victim->held_in_map = 0;
+      bytes_dropped += victim->bodylen;
+      microdesc_free(victim);
+    } else {
+      if (is_old) {
+        /* It's old, but it has held_by_nodes set.  That's not okay. */
+        /* Let's try to diagnose and fix #7164 . */
+        smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
+        const networkstatus_t *ns = networkstatus_get_latest_consensus();
+        long networkstatus_age = -1;
+        const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
+        if (ns) {
+          networkstatus_age = now - ns->valid_after;
+        }
+        log_warn(LD_BUG, "Microdescriptor seemed very old "
+                 "(last listed %d hours ago vs %d hour cutoff), but is still "
+                 "marked as being held by %d node(s). I found %d node(s) "
+                 "holding it. Current networkstatus is %ld hours old. "
+                 "Hashtable badness is %d.",
+                 (int)((now - (*mdp)->last_listed) / 3600),
+                 (int)((now - cutoff) / 3600),
+                 held_by_nodes,
+                 smartlist_len(nodes),
+                 networkstatus_age / 3600,
+                 ht_badness);
+
+        SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+          const char *rs_match = "No RS";
+          const char *rs_present = "";
+          if (node->rs) {
+            if (tor_memeq(node->rs->descriptor_digest,
+                          (*mdp)->digest, DIGEST256_LEN)) {
+              rs_match = "Microdesc digest in RS matches";
+            } else {
+              rs_match = "Microdesc digest in RS does match";
+            }
+            if (ns) {
+              /* This should be impossible, but let's see! */
+              rs_present = " RS not present in networkstatus.";
+              SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
+                if (rs == node->rs) {
+                  rs_present = " RS okay in networkstatus.";
+                }
+              });
+            }
+          }
+          log_warn(LD_BUG, "  [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
+                   node_sl_idx,
+                   hex_str(node->identity, DIGEST_LEN),
+                   node->md, node->rs, node->ri, rs_match, rs_present);
+        } SMARTLIST_FOREACH_END(node);
+        smartlist_free(nodes);
+        (*mdp)->last_listed = now;
+      }
+
+      ++kept;
+      mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
+    }
+  }
+
+  if (dropped) {
+    log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
+             dropped,dropped+kept);
+    cache->bytes_dropped += bytes_dropped;
+  }
+}
+
+static int
+should_rebuild_md_cache(microdesc_cache_t *cache)
+{
+    const size_t old_len =
+      cache->cache_content ? cache->cache_content->size : 0;
+    const size_t journal_len = cache->journal_len;
+    const size_t dropped = cache->bytes_dropped;
+
+    if (journal_len < 16384)
+      return 0; /* Don't bother, not enough has happened yet. */
+    if (dropped > (journal_len + old_len) / 3)
+      return 1; /* We could save 1/3 or more of the currently used space. */
+    if (journal_len > old_len / 2)
+      return 1; /* We should append to the regular file */
+
+    return 0;
+}
+
+/**
+ * Mark <b>md</b> as having no body, and release any storage previously held
+ * by its body.
+ */
+static void
+microdesc_wipe_body(microdesc_t *md)
+{
+  if (!md)
+    return;
+
+  if (md->saved_location != SAVED_IN_CACHE)
+    tor_free(md->body);
+
+  md->off = 0;
+  md->saved_location = SAVED_NOWHERE;
+  md->body = NULL;
+  md->bodylen = 0;
+  md->no_save = 1;
+}
+
+/** Regenerate the main cache file for <b>cache</b>, clear the journal file,
+ * and update every microdesc_t in the cache with pointers to its new
+ * location.  If <b>force</b> is true, do this unconditionally.  If
+ * <b>force</b> is false, do it only if we expect to save space on disk. */
+int
+microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
+{
+  open_file_t *open_file;
+  int fd = -1, res;
+  microdesc_t **mdp;
+  smartlist_t *wrote;
+  ssize_t size;
+  off_t off = 0, off_real;
+  int orig_size, new_size;
+
+  if (cache == NULL) {
+    cache = the_microdesc_cache;
+    if (cache == NULL)
+      return 0;
+  }
+
+  /* Remove dead descriptors */
+  microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
+
+  if (!force && !should_rebuild_md_cache(cache))
+    return 0;
+
+  log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
+
+  orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
+  orig_size += (int)cache->journal_len;
+
+  fd = start_writing_to_file(cache->cache_fname,
+                             OPEN_FLAGS_REPLACE|O_BINARY,
+                             0600, &open_file);
+  if (fd < 0)
+    return -1;
+
+  wrote = smartlist_new();
+
+  HT_FOREACH(mdp, microdesc_map, &cache->map) {
+    microdesc_t *md = *mdp;
+    size_t annotation_len;
+    if (md->no_save || !md->body)
+      continue;
+
+    size = dump_microdescriptor(fd, md, &annotation_len);
+    if (size < 0) {
+      microdesc_wipe_body(md);
+
+      /* rewind, in case it was a partial write. */
+      tor_fd_setpos(fd, off);
+      continue;
+    }
+    tor_assert(((size_t)size) == annotation_len + md->bodylen);
+    md->off = off + annotation_len;
+    off += size;
+    off_real = tor_fd_getpos(fd);
+    if (off_real != off) {
+      log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
+               "By my count, I'm at %"PRId64
+               ", but I should be at %"PRId64,
+               (off), (off_real));
+      if (off_real >= 0)
+        off = off_real;
+    }
+    if (md->saved_location != SAVED_IN_CACHE) {
+      tor_free(md->body);
+      md->saved_location = SAVED_IN_CACHE;
+    }
+    smartlist_add(wrote, md);
+  }
+
+  /* We must do this unmap _before_ we call finish_writing_to_file(), or
+   * windows will not actually replace the file. */
+  if (cache->cache_content) {
+    res = tor_munmap_file(cache->cache_content);
+    if (res != 0) {
+      log_warn(LD_FS,
+               "Failed to unmap old microdescriptor cache while rebuilding");
+    }
+    cache->cache_content = NULL;
+  }
+
+  if (finish_writing_to_file(open_file) < 0) {
+    log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
+             strerror(errno));
+    /* Okay. Let's prevent from making things worse elsewhere. */
+    cache->cache_content = NULL;
+    HT_FOREACH(mdp, microdesc_map, &cache->map) {
+      microdesc_t *md = *mdp;
+      if (md->saved_location == SAVED_IN_CACHE) {
+        microdesc_wipe_body(md);
+      }
+    }
+    smartlist_free(wrote);
+    return -1;
+  }
+
+  cache->cache_content = tor_mmap_file(cache->cache_fname);
+
+  if (!cache->cache_content && smartlist_len(wrote)) {
+    log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
+            cache->cache_fname);
+    smartlist_free(wrote);
+    return -1;
+  }
+  SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
+    tor_assert(md->saved_location == SAVED_IN_CACHE);
+    md->body = (char*)cache->cache_content->data + md->off;
+    if (PREDICT_UNLIKELY(
+             md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
+      /* XXXX once bug 2022 is solved, we can kill this block and turn it
+       * into just the tor_assert(fast_memeq) */
+      off_t avail = cache->cache_content->size - md->off;
+      char *bad_str;
+      tor_assert(avail >= 0);
+      bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
+      log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
+              " At offset %d, I expected to find a microdescriptor starting "
+              " with \"onion-key\".  Instead I got %s.",
+              (int)md->off, escaped(bad_str));
+      tor_free(bad_str);
+      tor_assert(fast_memeq(md->body, "onion-key", 9));
+    }
+  } SMARTLIST_FOREACH_END(md);
+
+  smartlist_free(wrote);
+
+  write_str_to_file(cache->journal_fname, "", 1);
+  cache->journal_len = 0;
+  cache->bytes_dropped = 0;
+
+  new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
+  log_info(LD_DIR, "Done rebuilding microdesc cache. "
+           "Saved %d bytes; %d still used.",
+           orig_size-new_size, new_size);
+
+  return 0;
+}
+
+/** Make sure that the reference count of every microdescriptor in cache is
+ * accurate. */
+void
+microdesc_check_counts(void)
+{
+  microdesc_t **mdp;
+  if (!the_microdesc_cache)
+    return;
+
+  HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
+    microdesc_t *md = *mdp;
+    unsigned int found=0;
+    const smartlist_t *nodes = nodelist_get_list();
+    SMARTLIST_FOREACH(nodes, node_t *, node, {
+        if (node->md == md) {
+          ++found;
+        }
+      });
+    tor_assert(found == md->held_by_nodes);
+  }
+}
+
+/** Deallocate a single microdescriptor.  Note: the microdescriptor MUST have
+ * previously been removed from the cache if it had ever been inserted. */
+void
+microdesc_free_(microdesc_t *md, const char *fname, int lineno)
+{
+  if (!md)
+    return;
+
+  /* Make sure that the microdesc was really removed from the appropriate data
+     structures. */
+  if (md->held_in_map) {
+    microdesc_cache_t *cache = get_microdesc_cache_noload();
+    microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
+    if (md2 == md) {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
+               "in microdesc_map", fname, lineno);
+      HT_REMOVE(microdesc_map, &cache->map, md);
+    } else {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
+               "set, but microdesc was not in the map.", fname, lineno);
+    }
+    tor_fragile_assert();
+  }
+  if (md->held_by_nodes) {
+    microdesc_cache_t *cache = get_microdesc_cache_noload();
+    int found=0;
+    const smartlist_t *nodes = nodelist_get_list();
+    const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
+    SMARTLIST_FOREACH(nodes, node_t *, node, {
+        if (node->md == md) {
+          ++found;
+          node->md = NULL;
+        }
+      });
+    if (found) {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
+               "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
+               fname, lineno, found, md->held_by_nodes, ht_badness);
+    } else {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
+               "set to %u, but md was not referenced by any nodes. "
+               "ht_badness == %d",
+               fname, lineno, md->held_by_nodes, ht_badness);
+    }
+    tor_fragile_assert();
+  }
+  //tor_assert(md->held_in_map == 0);
+  //tor_assert(md->held_by_nodes == 0);
+
+  if (md->onion_pkey)
+    crypto_pk_free(md->onion_pkey);
+  tor_free(md->onion_curve25519_pkey);
+  tor_free(md->ed25519_identity_pkey);
+  if (md->body && md->saved_location != SAVED_IN_CACHE)
+    tor_free(md->body);
+
+  if (md->family) {
+    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
+    smartlist_free(md->family);
+  }
+  short_policy_free(md->exit_policy);
+  short_policy_free(md->ipv6_exit_policy);
+
+  tor_free(md);
+}
+
+/** Free all storage held in the microdesc.c module. */
+void
+microdesc_free_all(void)
+{
+  if (the_microdesc_cache) {
+    microdesc_cache_clear(the_microdesc_cache);
+    tor_free(the_microdesc_cache->cache_fname);
+    tor_free(the_microdesc_cache->journal_fname);
+    tor_free(the_microdesc_cache);
+  }
+
+  if (outdated_dirserver_list) {
+    SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+    smartlist_free(outdated_dirserver_list);
+  }
+}
+
+/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
+ * <b>d</b>, return it.  Otherwise return NULL. */
+microdesc_t *
+microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
+{
+  microdesc_t *md, search;
+  if (!cache)
+    cache = get_microdesc_cache();
+  memcpy(search.digest, d, DIGEST256_LEN);
+  md = HT_FIND(microdesc_map, &cache->map, &search);
+  return md;
+}
+
+/** Return a smartlist of all the sha256 digest of the microdescriptors that
+ * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
+ * to internals of <b>ns</b>; you should not free the members of the resulting
+ * smartlist.  Omit all microdescriptors whose digest appear in <b>skip</b>. */
+smartlist_t *
+microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
+                                 int downloadable_only, digest256map_t *skip)
+{
+  smartlist_t *result = smartlist_new();
+  time_t now = time(NULL);
+  tor_assert(ns->flavor == FLAV_MICRODESC);
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
+      continue;
+    if (downloadable_only &&
+        !download_status_is_ready(&rs->dl_status, now))
+      continue;
+    if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
+      continue;
+    if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
+      continue;
+    /* XXXX Also skip if we're a noncache and wouldn't use this router.
+     * XXXX NM Microdesc
+     */
+    smartlist_add(result, rs->descriptor_digest);
+  } SMARTLIST_FOREACH_END(rs);
+  return result;
+}
+
+/** Launch download requests for microdescriptors as appropriate.
+ *
+ * Specifically, we should launch download requests if we are configured to
+ * download mirodescriptors, and there are some microdescriptors listed in the
+ * current microdesc consensus that we don't have, and either we never asked
+ * for them, or we failed to download them but we're willing to retry.
+ */
+void
+update_microdesc_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  networkstatus_t *consensus;
+  smartlist_t *missing;
+  digest256map_t *pending;
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  if (directory_too_idle_to_fetch_descriptors(options, now))
+    return;
+
+  consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+  if (!consensus)
+    return;
+
+  if (!we_fetch_microdescriptors(options))
+    return;
+
+  pending = digest256map_new();
+  list_pending_microdesc_downloads(pending);
+
+  missing = microdesc_list_missing_digest256(consensus,
+                                             get_microdesc_cache(),
+                                             1,
+                                             pending);
+  digest256map_free(pending, NULL);
+
+  launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
+                              missing, NULL, now);
+
+  smartlist_free(missing);
+}
+
+/** For every microdescriptor listed in the current microdescriptor consensus,
+ * update its last_listed field to be at least as recent as the publication
+ * time of the current microdescriptor consensus.
+ */
+void
+update_microdescs_from_networkstatus(time_t now)
+{
+  microdesc_cache_t *cache = get_microdesc_cache();
+  microdesc_t *md;
+  networkstatus_t *ns =
+    networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+
+  if (! ns)
+    return;
+
+  tor_assert(ns->flavor == FLAV_MICRODESC);
+
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
+    if (md && ns->valid_after > md->last_listed)
+      md->last_listed = ns->valid_after;
+  } SMARTLIST_FOREACH_END(rs);
+}
+
+/** Return true iff we should prefer to use microdescriptors rather than
+ * routerdescs for building circuits. */
+int
+we_use_microdescriptors_for_circuits(const or_options_t *options)
+{
+  if (options->UseMicrodescriptors == 0)
+    return 0; /* the user explicitly picked no */
+  return 1; /* yes and auto both mean yes */
+}
+
+/** Return true iff we should try to download microdescriptors at all. */
+int
+we_fetch_microdescriptors(const or_options_t *options)
+{
+  if (directory_caches_dir_info(options))
+    return 1;
+  if (options->FetchUselessDescriptors)
+    return 1;
+  return we_use_microdescriptors_for_circuits(options);
+}
+
+/** Return true iff we should try to download router descriptors at all. */
+int
+we_fetch_router_descriptors(const or_options_t *options)
+{
+  if (directory_caches_dir_info(options))
+    return 1;
+  if (options->FetchUselessDescriptors)
+    return 1;
+  return ! we_use_microdescriptors_for_circuits(options);
+}
+
+/** Return the consensus flavor we actually want to use to build circuits. */
+MOCK_IMPL(int,
+usable_consensus_flavor,(void))
+{
+  if (we_use_microdescriptors_for_circuits(get_options())) {
+    return FLAV_MICRODESC;
+  } else {
+    return FLAV_NS;
+  }
+}
diff --git a/src/feature/nodelist/microdesc.h b/src/feature/nodelist/microdesc.h
new file mode 100644
index 0000000000..f11b841cf1
--- /dev/null
+++ b/src/feature/nodelist/microdesc.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file microdesc.h
+ * \brief Header file for microdesc.c.
+ **/
+
+#ifndef TOR_MICRODESC_H
+#define TOR_MICRODESC_H
+
+microdesc_cache_t *get_microdesc_cache(void);
+
+void microdesc_check_counts(void);
+
+smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save, time_t listed_at,
+                        smartlist_t *requested_digests256);
+smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                        smartlist_t *descriptors, saved_location_t where,
+                        int no_save);
+
+void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force);
+int microdesc_cache_rebuild(microdesc_cache_t *cache, int force);
+int microdesc_cache_reload(microdesc_cache_t *cache);
+void microdesc_cache_clear(microdesc_cache_t *cache);
+
+microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
+                                                 const char *d);
+
+smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns,
+                                              microdesc_cache_t *cache,
+                                              int downloadable_only,
+                                              digest256map_t *skip);
+
+void microdesc_free_(microdesc_t *md, const char *fname, int line);
+#define microdesc_free(md) do {                 \
+    microdesc_free_((md), __FILE__, __LINE__);  \
+    (md) = NULL;                                \
+  } while (0)
+void microdesc_free_all(void);
+
+void update_microdesc_downloads(time_t now);
+void update_microdescs_from_networkstatus(time_t now);
+
+MOCK_DECL(int, usable_consensus_flavor,(void));
+int we_fetch_microdescriptors(const or_options_t *options);
+int we_fetch_router_descriptors(const or_options_t *options);
+int we_use_microdescriptors_for_circuits(const or_options_t *options);
+
+void microdesc_note_outdated_dirserver(const char *relay_digest);
+int microdesc_relay_is_outdated_dirserver(const char *relay_digest);
+void microdesc_reset_outdated_dirservers_list(void);
+
+#endif /* !defined(TOR_MICRODESC_H) */
+
diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h
new file mode 100644
index 0000000000..e9dc3e0174
--- /dev/null
+++ b/src/feature/nodelist/microdesc_st.h
@@ -0,0 +1,74 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef MICRODESC_ST_H
+#define MICRODESC_ST_H
+
+struct curve25519_public_key_t;
+struct ed25519_public_key_t;
+struct short_policy_t;
+
+/** A microdescriptor is the smallest amount of information needed to build a
+ * circuit through a router.  They are generated by the directory authorities,
+ * using information from the uploaded routerinfo documents.  They are not
+ * self-signed, but are rather authenticated by having their hash in a signed
+ * networkstatus document. */
+struct microdesc_t {
+  /** Hashtable node, used to look up the microdesc by its digest. */
+  HT_ENTRY(microdesc_t) node;
+
+  /* Cache information */
+
+  /**  When was this microdescriptor last listed in a consensus document?
+   * Once a microdesc has been unlisted long enough, we can drop it.
+   */
+  time_t last_listed;
+  /** Where is this microdescriptor currently stored? */
+  saved_location_bitfield_t saved_location : 3;
+  /** If true, do not attempt to cache this microdescriptor on disk. */
+  unsigned int no_save : 1;
+  /** If true, this microdesc has an entry in the microdesc_map */
+  unsigned int held_in_map : 1;
+  /** Reference count: how many node_ts have a reference to this microdesc? */
+  unsigned int held_by_nodes;
+
+  /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the
+   * microdescriptor in the cache. */
+  off_t off;
+
+  /* The string containing the microdesc. */
+
+  /** A pointer to the encoded body of the microdescriptor.  If the
+   * saved_location is SAVED_IN_CACHE, then the body is a pointer into an
+   * mmap'd region.  Otherwise, it is a malloc'd string.  The string might not
+   * be NUL-terminated; take the length from <b>bodylen</b>. */
+  char *body;
+  /** The length of the microdescriptor in <b>body</b>. */
+  size_t bodylen;
+  /** A SHA256-digest of the microdescriptor. */
+  char digest[DIGEST256_LEN];
+
+  /* Fields in the microdescriptor. */
+
+  /** As routerinfo_t.onion_pkey */
+  crypto_pk_t *onion_pkey;
+  /** As routerinfo_t.onion_curve25519_pkey */
+  struct curve25519_public_key_t *onion_curve25519_pkey;
+  /** Ed25519 identity key, if included. */
+  struct ed25519_public_key_t *ed25519_identity_pkey;
+  /** As routerinfo_t.ipv6_addr */
+  tor_addr_t ipv6_addr;
+  /** As routerinfo_t.ipv6_orport */
+  uint16_t ipv6_orport;
+  /** As routerinfo_t.family */
+  smartlist_t *family;
+  /** IPv4 exit policy summary */
+  struct short_policy_t *exit_policy;
+  /** IPv6 exit policy summary */
+  struct short_policy_t *ipv6_exit_policy;
+};
+
+#endif
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
new file mode 100644
index 0000000000..f91e46cdd7
--- /dev/null
+++ b/src/feature/nodelist/networkstatus.c
@@ -0,0 +1,2726 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file networkstatus.c
+ * \brief Functions and structures for handling networkstatus documents as a
+ * client or as a directory cache.
+ *
+ * A consensus networkstatus object is created by the directory
+ * authorities.  It authenticates a set of network parameters--most
+ * importantly, the list of all the relays in the network.  This list
+ * of relays is represented as an array of routerstatus_t objects.
+ *
+ * There are currently two flavors of consensus.  With the older "NS"
+ * flavor, each relay is associated with a digest of its router
+ * descriptor. Tor instances that use this consensus keep the list of
+ * router descriptors as routerinfo_t objects stored and managed in
+ * routerlist.c.  With the newer "microdesc" flavor, each relay is
+ * associated with a digest of the microdescriptor that the authorities
+ * made for it.  These are stored and managed in microdesc.c. Information
+ * about the router is divided between the the networkstatus and the
+ * microdescriptor according to the general rule that microdescriptors
+ * should hold information that changes much less frequently than the
+ * information in the networkstatus.
+ *
+ * Modern clients use microdescriptor networkstatuses.  Directory caches
+ * need to keep both kinds of networkstatus document, so they can serve them.
+ *
+ * This module manages fetching, holding, storing, updating, and
+ * validating networkstatus objects.  The download-and-validate process
+ * is slightly complicated by the fact that the keys you need to
+ * validate a consensus are stored in the authority certificates, which
+ * you might not have yet when you download the consensus.
+ */
+
+#define NETWORKSTATUS_PRIVATE
+#include "or/or.h"
+#include "or/bridges.h"
+#include "or/channel.h"
+#include "or/circuitmux.h"
+#include "or/circuitmux_ewma.h"
+#include "or/circuitstats.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/connection_or.h"
+#include "or/consdiffmgr.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/dos.h"
+#include "or/entrynodes.h"
+#include "or/hibernate.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/protover.h"
+#include "or/relay.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/scheduler.h"
+#include "or/transports.h"
+#include "or/torcert.h"
+#include "or/channelpadding.h"
+#include "or/voting_schedule.h"
+
+#include "or/dirauth/dirvote.h"
+#include "or/dirauth/mode.h"
+#include "or/dirauth/shared_random.h"
+
+#include "or/authority_cert_st.h"
+#include "or/dir_connection_st.h"
+#include "or/dir_server_st.h"
+#include "or/document_signature_st.h"
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+#include "or/ns_detached_signatures_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/vote_microdesc_hash_st.h"
+#include "or/vote_routerstatus_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** Most recently received and validated v3 "ns"-flavored consensus network
+ * status. */
+STATIC networkstatus_t *current_ns_consensus = NULL;
+
+/** Most recently received and validated v3 "microdesc"-flavored consensus
+ * network status. */
+STATIC networkstatus_t *current_md_consensus = NULL;
+
+/** A v3 consensus networkstatus that we've received, but which we don't
+ * have enough certificates to be happy about. */
+typedef struct consensus_waiting_for_certs_t {
+  /** The consensus itself. */
+  networkstatus_t *consensus;
+  /** The encoded version of the consensus, nul-terminated. */
+  char *body;
+  /** When did we set the current value of consensus_waiting_for_certs?  If
+   * this is too recent, we shouldn't try to fetch a new consensus for a
+   * little while, to give ourselves time to get certificates for this one. */
+  time_t set_at;
+  /** Set to 1 if we've been holding on to it for so long we should maybe
+   * treat it as being bad. */
+  int dl_failed;
+} consensus_waiting_for_certs_t;
+
+/** An array, for each flavor of consensus we might want, of consensuses that
+ * we have downloaded, but which we cannot verify due to having insufficient
+ * authority certificates. */
+static consensus_waiting_for_certs_t
+       consensus_waiting_for_certs[N_CONSENSUS_FLAVORS];
+
+/** A time before which we shouldn't try to replace the current consensus:
+ * this will be at some point after the next consensus becomes valid, but
+ * before the current consensus becomes invalid. */
+static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS];
+/** Download status for the current consensus networkstatus. */
+static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] =
+  {
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
+  };
+
+#define N_CONSENSUS_BOOTSTRAP_SCHEDULES 2
+#define CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY 0
+#define CONSENSUS_BOOTSTRAP_SOURCE_ANY_DIRSERVER  1
+
+/* Using DL_SCHED_INCREMENT_ATTEMPT on these schedules means that
+ * download_status_increment_failure won't increment these entries.
+ * However, any bootstrap connection failures that occur after we have
+ * a valid consensus will count against the failure counts on the non-bootstrap
+ * schedules. There should only be one of these, as all the others will have
+ * been cancelled. (This doesn't seem to be a significant issue.) */
+static download_status_t
+              consensus_bootstrap_dl_status[N_CONSENSUS_BOOTSTRAP_SCHEDULES] =
+  {
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
+    /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
+  };
+
+/** True iff we have logged a warning about this OR's version being older than
+ * listed by the authorities. */
+static int have_warned_about_old_version = 0;
+/** True iff we have logged a warning about this OR's version being newer than
+ * listed by the authorities. */
+static int have_warned_about_new_version = 0;
+
+static void update_consensus_bootstrap_multiple_downloads(
+                                                  time_t now,
+                                                  const or_options_t *options);
+static int networkstatus_check_required_protocols(const networkstatus_t *ns,
+                                                  int client_mode,
+                                                  char **warning_out);
+
+/** Forget that we've warned about anything networkstatus-related, so we will
+ * give fresh warnings if the same behavior happens again. */
+void
+networkstatus_reset_warnings(void)
+{
+  SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node,
+                    node->name_lookup_warned = 0);
+
+  have_warned_about_old_version = 0;
+  have_warned_about_new_version = 0;
+}
+
+/** Reset the descriptor download failure count on all networkstatus docs, so
+ * that we can retry any long-failed documents immediately.
+ */
+void
+networkstatus_reset_download_failures(void)
+{
+  int i;
+
+  log_debug(LD_GENERAL,
+            "In networkstatus_reset_download_failures()");
+
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i)
+    download_status_reset(&consensus_dl_status[i]);
+
+  for (i=0; i < N_CONSENSUS_BOOTSTRAP_SCHEDULES; ++i)
+    download_status_reset(&consensus_bootstrap_dl_status[i]);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static char *
+networkstatus_read_cached_consensus_impl(int flav,
+                                         const char *flavorname,
+                                         int unverified_consensus)
+{
+  char buf[128];
+  const char *prefix;
+  if (unverified_consensus) {
+    prefix = "unverified";
+  } else {
+    prefix = "cached";
+  }
+  if (flav == FLAV_NS) {
+    tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix);
+  } else {
+    tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
+  }
+
+  char *filename = get_cachedir_fname(buf);
+  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  return result;
+}
+
+/** Return a new string containing the current cached consensus of flavor
+ * <b>flavorname</b>. */
+char *
+networkstatus_read_cached_consensus(const char *flavorname)
+ {
+  int flav = networkstatus_parse_flavor_name(flavorname);
+  if (flav < 0)
+    return NULL;
+  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+}
+
+/** Read every cached v3 consensus networkstatus from the disk. */
+int
+router_reload_consensus_networkstatus(void)
+{
+  const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS;
+  int flav;
+
+  /* FFFF Suppress warnings if cached consensus is bad? */
+  for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavor = networkstatus_get_flavor_name(flav);
+    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
+    if (s) {
+      if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
+        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
+                 flavor);
+      }
+      tor_free(s);
+    }
+
+    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
+    if (s) {
+      if (networkstatus_set_current_consensus(s, flavor,
+                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
+                                     NULL)) {
+        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
+                 "from cache", flavor);
+      }
+      tor_free(s);
+    }
+  }
+
+  update_certificate_downloads(time(NULL));
+
+  routers_update_all_from_networkstatus(time(NULL), 3);
+  update_microdescs_from_networkstatus(time(NULL));
+
+  return 0;
+}
+
+/** Free all storage held by the vote_routerstatus object <b>rs</b>. */
+void
+vote_routerstatus_free_(vote_routerstatus_t *rs)
+{
+  vote_microdesc_hash_t *h, *next;
+  if (!rs)
+    return;
+  tor_free(rs->version);
+  tor_free(rs->protocols);
+  tor_free(rs->status.exitsummary);
+  for (h = rs->microdesc; h; h = next) {
+    tor_free(h->microdesc_hash_line);
+    next = h->next;
+    tor_free(h);
+  }
+  tor_free(rs);
+}
+
+/** Free all storage held by the routerstatus object <b>rs</b>. */
+void
+routerstatus_free_(routerstatus_t *rs)
+{
+  if (!rs)
+    return;
+  tor_free(rs->exitsummary);
+  tor_free(rs);
+}
+
+/** Free all storage held in <b>sig</b> */
+void
+document_signature_free_(document_signature_t *sig)
+{
+  tor_free(sig->signature);
+  tor_free(sig);
+}
+
+/** Return a newly allocated copy of <b>sig</b> */
+document_signature_t *
+document_signature_dup(const document_signature_t *sig)
+{
+  document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t));
+  if (r->signature)
+    r->signature = tor_memdup(sig->signature, sig->signature_len);
+  return r;
+}
+
+/** Free all storage held in <b>ns</b>. */
+void
+networkstatus_vote_free_(networkstatus_t *ns)
+{
+  if (!ns)
+    return;
+
+  tor_free(ns->client_versions);
+  tor_free(ns->server_versions);
+  tor_free(ns->recommended_client_protocols);
+  tor_free(ns->recommended_relay_protocols);
+  tor_free(ns->required_client_protocols);
+  tor_free(ns->required_relay_protocols);
+
+  if (ns->known_flags) {
+    SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c));
+    smartlist_free(ns->known_flags);
+  }
+  if (ns->weight_params) {
+    SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c));
+    smartlist_free(ns->weight_params);
+  }
+  if (ns->net_params) {
+    SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c));
+    smartlist_free(ns->net_params);
+  }
+  if (ns->supported_methods) {
+    SMARTLIST_FOREACH(ns->supported_methods, char *, c, tor_free(c));
+    smartlist_free(ns->supported_methods);
+  }
+  if (ns->package_lines) {
+    SMARTLIST_FOREACH(ns->package_lines, char *, c, tor_free(c));
+    smartlist_free(ns->package_lines);
+  }
+  if (ns->voters) {
+    SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) {
+      tor_free(voter->nickname);
+      tor_free(voter->address);
+      tor_free(voter->contact);
+      if (voter->sigs) {
+        SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                          document_signature_free(sig));
+        smartlist_free(voter->sigs);
+      }
+      tor_free(voter);
+    } SMARTLIST_FOREACH_END(voter);
+    smartlist_free(ns->voters);
+  }
+  authority_cert_free(ns->cert);
+
+  if (ns->routerstatus_list) {
+    if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) {
+      SMARTLIST_FOREACH(ns->routerstatus_list, vote_routerstatus_t *, rs,
+                        vote_routerstatus_free(rs));
+    } else {
+      SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *, rs,
+                        routerstatus_free(rs));
+    }
+
+    smartlist_free(ns->routerstatus_list);
+  }
+
+  digestmap_free(ns->desc_digest_map, NULL);
+
+  if (ns->sr_info.commits) {
+    dirvote_clear_commits(ns);
+  }
+  tor_free(ns->sr_info.previous_srv);
+  tor_free(ns->sr_info.current_srv);
+
+  memwipe(ns, 11, sizeof(*ns));
+  tor_free(ns);
+}
+
+/** Return the voter info from <b>vote</b> for the voter whose identity digest
+ * is <b>identity</b>, or NULL if no such voter is associated with
+ * <b>vote</b>. */
+networkstatus_voter_info_t *
+networkstatus_get_voter_by_id(networkstatus_t *vote,
+                              const char *identity)
+{
+  if (!vote || !vote->voters)
+    return NULL;
+  SMARTLIST_FOREACH(vote->voters, networkstatus_voter_info_t *, voter,
+    if (fast_memeq(voter->identity_digest, identity, DIGEST_LEN))
+      return voter);
+  return NULL;
+}
+
+/** Return the signature made by <b>voter</b> using the algorithm
+ * <b>alg</b>, or NULL if none is found. */
+document_signature_t *
+networkstatus_get_voter_sig_by_alg(const networkstatus_voter_info_t *voter,
+                                   digest_algorithm_t alg)
+{
+  if (!voter->sigs)
+    return NULL;
+  SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                    if (sig->alg == alg)
+                    return sig);
+  return NULL;
+}
+
+/** Check whether the signature <b>sig</b> is correctly signed with the
+ * signing key in <b>cert</b>.  Return -1 if <b>cert</b> doesn't match the
+ * signing key; otherwise set the good_signature or bad_signature flag on
+ * <b>voter</b>, and return 0. */
+int
+networkstatus_check_document_signature(const networkstatus_t *consensus,
+                                       document_signature_t *sig,
+                                       const authority_cert_t *cert)
+{
+  char key_digest[DIGEST_LEN];
+  const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN;
+  char *signed_digest;
+  size_t signed_digest_len;
+
+  if (crypto_pk_get_digest(cert->signing_key, key_digest)<0)
+    return -1;
+  if (tor_memneq(sig->signing_key_digest, key_digest, DIGEST_LEN) ||
+      tor_memneq(sig->identity_digest, cert->cache_info.identity_digest,
+                 DIGEST_LEN))
+    return -1;
+
+  if (authority_cert_is_blacklisted(cert)) {
+    /* We implement blacklisting for authority signing keys by treating
+     * all their signatures as always bad. That way we don't get into
+     * crazy loops of dropping and re-fetching signatures. */
+    log_warn(LD_DIR, "Ignoring a consensus signature made with deprecated"
+             " signing key %s",
+             hex_str(cert->signing_key_digest, DIGEST_LEN));
+    sig->bad_signature = 1;
+    return 0;
+  }
+
+  signed_digest_len = crypto_pk_keysize(cert->signing_key);
+  signed_digest = tor_malloc(signed_digest_len);
+  if (crypto_pk_public_checksig(cert->signing_key,
+                                signed_digest,
+                                signed_digest_len,
+                                sig->signature,
+                                sig->signature_len) < dlen ||
+      tor_memneq(signed_digest, consensus->digests.d[sig->alg], dlen)) {
+    log_warn(LD_DIR, "Got a bad signature on a networkstatus vote");
+    sig->bad_signature = 1;
+  } else {
+    sig->good_signature = 1;
+  }
+  tor_free(signed_digest);
+  return 0;
+}
+
+/** Given a v3 networkstatus consensus in <b>consensus</b>, check every
+ * as-yet-unchecked signature on <b>consensus</b>.  Return 1 if there is a
+ * signature from every recognized authority on it, 0 if there are
+ * enough good signatures from recognized authorities on it, -1 if we might
+ * get enough good signatures by fetching missing certificates, and -2
+ * otherwise.  Log messages at INFO or WARN: if <b>warn</b> is over 1, warn
+ * about every problem; if warn is at least 1, warn only if we can't get
+ * enough signatures; if warn is negative, log nothing at all. */
+int
+networkstatus_check_consensus_signature(networkstatus_t *consensus,
+                                        int warn)
+{
+  int n_good = 0;
+  int n_missing_key = 0, n_dl_failed_key = 0;
+  int n_bad = 0;
+  int n_unknown = 0;
+  int n_no_signature = 0;
+  int n_v3_authorities = get_n_authorities(V3_DIRINFO);
+  int n_required = n_v3_authorities/2 + 1;
+  smartlist_t *list_good = smartlist_new();
+  smartlist_t *list_no_signature = smartlist_new();
+  smartlist_t *need_certs_from = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  smartlist_t *missing_authorities = smartlist_new();
+  int severity;
+  time_t now = time(NULL);
+
+  tor_assert(consensus->type == NS_TYPE_CONSENSUS);
+
+  SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *,
+                          voter) {
+    int good_here = 0;
+    int bad_here = 0;
+    int unknown_here = 0;
+    int missing_key_here = 0, dl_failed_key_here = 0;
+    SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+      if (!sig->good_signature && !sig->bad_signature &&
+          sig->signature) {
+        /* we can try to check the signature. */
+        int is_v3_auth = trusteddirserver_get_by_v3_auth_digest(
+                                              sig->identity_digest) != NULL;
+        authority_cert_t *cert =
+          authority_cert_get_by_digests(sig->identity_digest,
+                                        sig->signing_key_digest);
+        tor_assert(tor_memeq(sig->identity_digest, voter->identity_digest,
+                           DIGEST_LEN));
+
+        if (!is_v3_auth) {
+          smartlist_add(unrecognized, voter);
+          ++unknown_here;
+          continue;
+        } else if (!cert || cert->expires < now) {
+          smartlist_add(need_certs_from, voter);
+          ++missing_key_here;
+          if (authority_cert_dl_looks_uncertain(sig->identity_digest))
+            ++dl_failed_key_here;
+          continue;
+        }
+        if (networkstatus_check_document_signature(consensus, sig, cert) < 0) {
+          smartlist_add(need_certs_from, voter);
+          ++missing_key_here;
+          if (authority_cert_dl_looks_uncertain(sig->identity_digest))
+            ++dl_failed_key_here;
+          continue;
+        }
+      }
+      if (sig->good_signature)
+        ++good_here;
+      else if (sig->bad_signature)
+        ++bad_here;
+    } SMARTLIST_FOREACH_END(sig);
+
+    if (good_here) {
+      ++n_good;
+      smartlist_add(list_good, voter->nickname);
+    } else if (bad_here) {
+      ++n_bad;
+    } else if (missing_key_here) {
+      ++n_missing_key;
+      if (dl_failed_key_here)
+        ++n_dl_failed_key;
+    } else if (unknown_here) {
+      ++n_unknown;
+    } else {
+      ++n_no_signature;
+      smartlist_add(list_no_signature, voter->nickname);
+    }
+  } SMARTLIST_FOREACH_END(voter);
+
+  /* Now see whether we're missing any voters entirely. */
+  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
+                    dir_server_t *, ds,
+    {
+      if ((ds->type & V3_DIRINFO) &&
+          !networkstatus_get_voter_by_id(consensus, ds->v3_identity_digest))
+        smartlist_add(missing_authorities, ds);
+    });
+
+  if (warn > 1 || (warn >= 0 &&
+                   (n_good + n_missing_key - n_dl_failed_key < n_required))) {
+    severity = LOG_WARN;
+  } else {
+    severity = LOG_INFO;
+  }
+
+  if (warn >= 0) {
+    SMARTLIST_FOREACH(unrecognized, networkstatus_voter_info_t *, voter,
+      {
+        tor_log(severity, LD_DIR, "Consensus includes unrecognized authority "
+                 "'%s' at %s:%d (contact %s; identity %s)",
+                 voter->nickname, voter->address, (int)voter->dir_port,
+                 voter->contact?voter->contact:"n/a",
+                 hex_str(voter->identity_digest, DIGEST_LEN));
+      });
+    SMARTLIST_FOREACH(need_certs_from, networkstatus_voter_info_t *, voter,
+      {
+        tor_log(severity, LD_DIR, "Looks like we need to download a new "
+                 "certificate from authority '%s' at %s:%d (contact %s; "
+                 "identity %s)",
+                 voter->nickname, voter->address, (int)voter->dir_port,
+                 voter->contact?voter->contact:"n/a",
+                 hex_str(voter->identity_digest, DIGEST_LEN));
+      });
+    SMARTLIST_FOREACH(missing_authorities, dir_server_t *, ds,
+      {
+        tor_log(severity, LD_DIR, "Consensus does not include configured "
+                 "authority '%s' at %s:%d (identity %s)",
+                 ds->nickname, ds->address, (int)ds->dir_port,
+                 hex_str(ds->v3_identity_digest, DIGEST_LEN));
+      });
+    {
+      char *joined;
+      smartlist_t *sl = smartlist_new();
+      char *tmp = smartlist_join_strings(list_good, " ", 0, NULL);
+      smartlist_add_asprintf(sl,
+                   "A consensus needs %d good signatures from recognized "
+                   "authorities for us to accept it. "
+                   "This %s one has %d (%s).",
+                   n_required,
+                   networkstatus_get_flavor_name(consensus->flavor),
+                   n_good, tmp);
+      tor_free(tmp);
+      if (n_no_signature) {
+        tmp = smartlist_join_strings(list_no_signature, " ", 0, NULL);
+        smartlist_add_asprintf(sl,
+                     "%d (%s) of the authorities we know didn't sign it.",
+                     n_no_signature, tmp);
+        tor_free(tmp);
+      }
+      if (n_unknown) {
+        smartlist_add_asprintf(sl,
+                      "It has %d signatures from authorities we don't "
+                      "recognize.", n_unknown);
+      }
+      if (n_bad) {
+        smartlist_add_asprintf(sl, "%d of the signatures on it didn't verify "
+                      "correctly.", n_bad);
+      }
+      if (n_missing_key) {
+        smartlist_add_asprintf(sl,
+                      "We were unable to check %d of the signatures, "
+                      "because we were missing the keys.", n_missing_key);
+      }
+      joined = smartlist_join_strings(sl, " ", 0, NULL);
+      tor_log(severity, LD_DIR, "%s", joined);
+      tor_free(joined);
+      SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+      smartlist_free(sl);
+    }
+  }
+
+  smartlist_free(list_good);
+  smartlist_free(list_no_signature);
+  smartlist_free(unrecognized);
+  smartlist_free(need_certs_from);
+  smartlist_free(missing_authorities);
+
+  if (n_good == n_v3_authorities)
+    return 1;
+  else if (n_good >= n_required)
+    return 0;
+  else if (n_good + n_missing_key >= n_required)
+    return -1;
+  else
+    return -2;
+}
+
+/** How far in the future do we allow a network-status to get before removing
+ * it? (seconds) */
+#define NETWORKSTATUS_ALLOW_SKEW (24*60*60)
+
+/** Helper for bsearching a list of routerstatus_t pointers: compare a
+ * digest in the key to the identity digest of a routerstatus_t. */
+int
+compare_digest_to_routerstatus_entry(const void *_key, const void **_member)
+{
+  const char *key = _key;
+  const routerstatus_t *rs = *_member;
+  return tor_memcmp(key, rs->identity_digest, DIGEST_LEN);
+}
+
+/** Helper for bsearching a list of routerstatus_t pointers: compare a
+ * digest in the key to the identity digest of a routerstatus_t. */
+int
+compare_digest_to_vote_routerstatus_entry(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const vote_routerstatus_t *vrs = *_member;
+  return tor_memcmp(key, vrs->status.identity_digest, DIGEST_LEN);
+}
+
+/** As networkstatus_find_entry, but do not return a const pointer */
+routerstatus_t *
+networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest)
+{
+  return smartlist_bsearch(ns->routerstatus_list, digest,
+                           compare_digest_to_routerstatus_entry);
+}
+
+/** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or
+ * NULL if none was found. */
+const routerstatus_t *
+networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+{
+  return networkstatus_vote_find_mutable_entry(ns, digest);
+}
+
+/*XXXX MOVE make this static once functions are moved into this file. */
+/** Search the routerstatuses in <b>ns</b> for one whose identity digest is
+ * <b>digest</b>.  Return value and set *<b>found_out</b> as for
+ * smartlist_bsearch_idx(). */
+int
+networkstatus_vote_find_entry_idx(networkstatus_t *ns,
+                                  const char *digest, int *found_out)
+{
+  return smartlist_bsearch_idx(ns->routerstatus_list, digest,
+                               compare_digest_to_routerstatus_entry,
+                               found_out);
+}
+
+/** As router_get_consensus_status_by_descriptor_digest, but does not return
+ * a const pointer. */
+MOCK_IMPL(routerstatus_t *,
+router_get_mutable_consensus_status_by_descriptor_digest,(
+                                                 networkstatus_t *consensus,
+                                                 const char *digest))
+{
+  if (!consensus)
+    consensus = networkstatus_get_latest_consensus();
+  if (!consensus)
+    return NULL;
+  if (!consensus->desc_digest_map) {
+    digestmap_t *m = consensus->desc_digest_map = digestmap_new();
+    SMARTLIST_FOREACH(consensus->routerstatus_list,
+                      routerstatus_t *, rs,
+     {
+       digestmap_set(m, rs->descriptor_digest, rs);
+     });
+  }
+  return digestmap_get(consensus->desc_digest_map, digest);
+}
+
+/** Return the consensus view of the status of the router whose current
+ * <i>descriptor</i> digest in <b>consensus</b> is <b>digest</b>, or NULL if
+ * no such router is known. */
+const routerstatus_t *
+router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus,
+                                                 const char *digest)
+{
+  return router_get_mutable_consensus_status_by_descriptor_digest(
+                                                          consensus, digest);
+}
+
+/** Return a smartlist of all router descriptor digests in a consensus */
+static smartlist_t *
+router_get_descriptor_digests_in_consensus(networkstatus_t *consensus)
+{
+  smartlist_t *result = smartlist_new();
+  digestmap_iter_t *i;
+  const char *digest;
+  void *rs;
+  char *digest_tmp;
+
+  for (i = digestmap_iter_init(consensus->desc_digest_map);
+       !(digestmap_iter_done(i));
+       i = digestmap_iter_next(consensus->desc_digest_map, i)) {
+    digestmap_iter_get(i, &digest, &rs);
+    digest_tmp = tor_malloc(DIGEST_LEN);
+    memcpy(digest_tmp, digest, DIGEST_LEN);
+    smartlist_add(result, digest_tmp);
+  }
+
+  return result;
+}
+
+/** Return a smartlist of all router descriptor digests in the current
+ * consensus */
+MOCK_IMPL(smartlist_t *,
+router_get_descriptor_digests,(void))
+{
+  smartlist_t *result = NULL;
+
+  if (current_ns_consensus) {
+    result =
+      router_get_descriptor_digests_in_consensus(current_ns_consensus);
+  }
+
+  return result;
+}
+
+/** Given the digest of a router descriptor, return its current download
+ * status, or NULL if the digest is unrecognized. */
+MOCK_IMPL(download_status_t *,
+router_get_dl_status_by_descriptor_digest,(const char *d))
+{
+  routerstatus_t *rs;
+  if (!current_ns_consensus)
+    return NULL;
+  if ((rs = router_get_mutable_consensus_status_by_descriptor_digest(
+                                              current_ns_consensus, d)))
+    return &rs->dl_status;
+
+  return NULL;
+}
+
+/** As router_get_consensus_status_by_id, but do not return a const pointer */
+routerstatus_t *
+router_get_mutable_consensus_status_by_id(const char *digest)
+{
+  const networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (!ns)
+    return NULL;
+  smartlist_t *rslist = ns->routerstatus_list;
+  return smartlist_bsearch(rslist, digest,
+                           compare_digest_to_routerstatus_entry);
+}
+
+/** Return the consensus view of the status of the router whose identity
+ * digest is <b>digest</b>, or NULL if we don't know about any such router. */
+const routerstatus_t *
+router_get_consensus_status_by_id(const char *digest)
+{
+  return router_get_mutable_consensus_status_by_id(digest);
+}
+
+/** How frequently do directory authorities re-download fresh networkstatus
+ * documents? */
+#define AUTHORITY_NS_CACHE_INTERVAL (10*60)
+
+/** How frequently do non-authority directory caches re-download fresh
+ * networkstatus documents? */
+#define NONAUTHORITY_NS_CACHE_INTERVAL (60*60)
+
+/** Return true iff, given the options listed in <b>options</b>, <b>flavor</b>
+ *  is the flavor of a consensus networkstatus that we would like to fetch.
+ *
+ * For certificate fetches, use we_want_to_fetch_unknown_auth_certs, and
+ * for serving fetched documents, use directory_caches_dir_info. */
+int
+we_want_to_fetch_flavor(const or_options_t *options, int flavor)
+{
+  if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
+    /* This flavor is crazy; we don't want it */
+    /*XXXX handle unrecognized flavors later */
+    return 0;
+  }
+  if (authdir_mode_v3(options) || directory_caches_dir_info(options)) {
+    /* We want to serve all flavors to others, regardless if we would use
+     * it ourselves. */
+    return 1;
+  }
+  if (options->FetchUselessDescriptors) {
+    /* In order to get all descriptors, we need to fetch all consensuses. */
+    return 1;
+  }
+  /* Otherwise, we want the flavor only if we want to use it to build
+   * circuits. */
+  return flavor == usable_consensus_flavor();
+}
+
+/** Return true iff, given the options listed in <b>options</b>, we would like
+ * to fetch and store unknown authority certificates.
+ *
+ * For consensus and descriptor fetches, use we_want_to_fetch_flavor, and
+ * for serving fetched certificates, use directory_caches_unknown_auth_certs.
+ */
+int
+we_want_to_fetch_unknown_auth_certs(const or_options_t *options)
+{
+  if (authdir_mode_v3(options) ||
+      directory_caches_unknown_auth_certs((options))) {
+    /* We want to serve all certs to others, regardless if we would use
+     * them ourselves. */
+    return 1;
+  }
+  if (options->FetchUselessDescriptors) {
+    /* Unknown certificates are definitely useless. */
+    return 1;
+  }
+  /* Otherwise, don't fetch unknown certificates. */
+  return 0;
+}
+
+/** How long will we hang onto a possibly live consensus for which we're
+ * fetching certs before we check whether there is a better one? */
+#define DELAY_WHILE_FETCHING_CERTS (20*60)
+
+/** What is the minimum time we need to have waited fetching certs, before we
+ * increment the consensus download schedule on failure? */
+#define MIN_DELAY_FOR_FETCH_CERT_STATUS_FAILURE (1*60)
+
+/* Check if a downloaded consensus flavor should still wait for certificates
+ * to download now. If we decide not to wait, check if enough time has passed
+ * to consider the certificate download failure a separate failure. If so,
+ * fail dls.
+ * If waiting for certificates to download, return 1. If not, return 0. */
+static int
+check_consensus_waiting_for_certs(int flavor, time_t now,
+                                  download_status_t *dls)
+{
+  consensus_waiting_for_certs_t *waiting;
+
+  /* We should always have a known flavor, because we_want_to_fetch_flavor()
+   * filters out unknown flavors. */
+  tor_assert(flavor >= 0 && flavor < N_CONSENSUS_FLAVORS);
+
+  waiting = &consensus_waiting_for_certs[flavor];
+  if (waiting->consensus) {
+    /* XXXX make sure this doesn't delay sane downloads. */
+    if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now &&
+        waiting->consensus->valid_until > now) {
+      return 1;
+    } else {
+      if (!waiting->dl_failed) {
+        if (waiting->set_at + MIN_DELAY_FOR_FETCH_CERT_STATUS_FAILURE > now) {
+          download_status_failed(dls, 0);
+        }
+        waiting->dl_failed=1;
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** If we want to download a fresh consensus, launch a new download as
+ * appropriate. */
+static void
+update_consensus_networkstatus_downloads(time_t now)
+{
+  int i;
+  const or_options_t *options = get_options();
+  const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping(
+                                                                        now);
+  const int use_multi_conn =
+    networkstatus_consensus_can_use_multiple_directories(options);
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
+    /* XXXX need some way to download unknown flavors if we are caching. */
+    const char *resource;
+    networkstatus_t *c;
+    int max_in_progress_conns = 1;
+
+    if (! we_want_to_fetch_flavor(options, i))
+      continue;
+
+    c = networkstatus_get_latest_consensus_by_flavor(i);
+    if (! (c && c->valid_after <= now && now <= c->valid_until)) {
+      /* No live consensus? Get one now!*/
+      time_to_download_next_consensus[i] = now;
+    }
+
+    if (time_to_download_next_consensus[i] > now)
+      continue; /* Wait until the current consensus is older. */
+
+    resource = networkstatus_get_flavor_name(i);
+
+    /* Check if we already have enough connections in progress */
+    if (we_are_bootstrapping && use_multi_conn) {
+      max_in_progress_conns =
+        options->ClientBootstrapConsensusMaxInProgressTries;
+    }
+    if (connection_dir_count_by_purpose_and_resource(
+                                                  DIR_PURPOSE_FETCH_CONSENSUS,
+                                                  resource)
+        >= max_in_progress_conns) {
+      continue;
+    }
+
+    /* Check if we want to launch another download for a usable consensus.
+     * Only used during bootstrap. */
+    if (we_are_bootstrapping && use_multi_conn
+        && i == usable_consensus_flavor()) {
+
+      /* Check if we're already downloading a usable consensus */
+      if (networkstatus_consensus_is_already_downloading(resource))
+        continue;
+
+      /* Make multiple connections for a bootstrap consensus download. */
+      update_consensus_bootstrap_multiple_downloads(now, options);
+    } else {
+      /* Check if we failed downloading a consensus too recently */
+
+      /* Let's make sure we remembered to update consensus_dl_status */
+      tor_assert(consensus_dl_status[i].schedule == DL_SCHED_CONSENSUS);
+
+      if (!download_status_is_ready(&consensus_dl_status[i], now)) {
+        continue;
+      }
+
+      /** Check if we're waiting for certificates to download. If we are,
+       * launch download for missing directory authority certificates. */
+      if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) {
+        update_certificate_downloads(now);
+        continue;
+      }
+
+      /* Try the requested attempt */
+      log_info(LD_DIR, "Launching %s standard networkstatus consensus "
+               "download.", networkstatus_get_flavor_name(i));
+      directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+                                   ROUTER_PURPOSE_GENERAL, resource,
+                                   PDS_RETRY_IF_NO_SERVERS,
+                                   consensus_dl_status[i].want_authority);
+    }
+  }
+}
+
+/** When we're bootstrapping, launch one or more consensus download
+ * connections, if schedule indicates connection(s) should be made after now.
+ * If is_authority, connect to an authority, otherwise, use a fallback
+ * directory mirror.
+ */
+static void
+update_consensus_bootstrap_attempt_downloads(
+                                      time_t now,
+                                      download_status_t *dls,
+                                      download_want_authority_t want_authority)
+{
+  const char *resource = networkstatus_get_flavor_name(
+                                                  usable_consensus_flavor());
+
+  /* Let's make sure we remembered to update schedule */
+  tor_assert(dls->schedule == DL_SCHED_CONSENSUS);
+
+  /* Allow for multiple connections in the same second, if the schedule value
+   * is 0. */
+  while (download_status_is_ready(dls, now)) {
+    log_info(LD_DIR, "Launching %s bootstrap %s networkstatus consensus "
+             "download.", resource, (want_authority == DL_WANT_AUTHORITY
+                                     ? "authority"
+                                     : "mirror"));
+
+    directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+                                 ROUTER_PURPOSE_GENERAL, resource,
+                                 PDS_RETRY_IF_NO_SERVERS, want_authority);
+    /* schedule the next attempt */
+    download_status_increment_attempt(dls, resource, now);
+  }
+}
+
+/** If we're bootstrapping, check the connection schedules and see if we want
+ * to make additional, potentially concurrent, consensus download
+ * connections.
+ * Only call when bootstrapping, and when we want to make additional
+ * connections. Only nodes that satisfy
+ * networkstatus_consensus_can_use_multiple_directories make additional
+ * connections.
+ */
+static void
+update_consensus_bootstrap_multiple_downloads(time_t now,
+                                              const or_options_t *options)
+{
+  const int usable_flavor = usable_consensus_flavor();
+
+  /* make sure we can use multiple connections */
+  if (!networkstatus_consensus_can_use_multiple_directories(options)) {
+    return;
+  }
+
+  /* Launch concurrent consensus download attempt(s) based on the mirror and
+   * authority schedules. Try the mirror first - this makes it slightly more
+   * likely that we'll connect to the fallback first, and then end the
+   * authority connection attempt. */
+
+  /* If a consensus download fails because it's waiting for certificates,
+   * we'll fail both the authority and fallback schedules. This is better than
+   * failing only one of the schedules, and having the other continue
+   * unchecked.
+   */
+
+  /* If we don't have or can't use extra fallbacks, don't try them. */
+  if (networkstatus_consensus_can_use_extra_fallbacks(options)) {
+    download_status_t *dls_f =
+      &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_ANY_DIRSERVER];
+
+    if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_f)) {
+      /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
+      update_consensus_bootstrap_attempt_downloads(now, dls_f,
+                                                   DL_WANT_ANY_DIRSERVER);
+    }
+  }
+
+  /* Now try an authority. */
+  download_status_t *dls_a =
+    &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY];
+
+  if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_a)) {
+    update_consensus_bootstrap_attempt_downloads(now, dls_a,
+                                                 DL_WANT_AUTHORITY);
+  }
+}
+
+/** Called when an attempt to download a consensus fails: note that the
+ * failure occurred, and possibly retry. */
+void
+networkstatus_consensus_download_failed(int status_code, const char *flavname)
+{
+  int flav = networkstatus_parse_flavor_name(flavname);
+  if (flav >= 0) {
+    tor_assert(flav < N_CONSENSUS_FLAVORS);
+    /* XXXX handle unrecognized flavors */
+    download_status_failed(&consensus_dl_status[flav], status_code);
+    /* Retry immediately, if appropriate. */
+    update_consensus_networkstatus_downloads(time(NULL));
+  }
+}
+
+/** How long do we (as a cache) wait after a consensus becomes non-fresh
+ * before trying to fetch another? */
+#define CONSENSUS_MIN_SECONDS_BEFORE_CACHING 120
+
+/** Update the time at which we'll consider replacing the current
+ * consensus of flavor <b>flav</b> */
+static void
+update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
+{
+  const or_options_t *options = get_options();
+  networkstatus_t *c = networkstatus_get_latest_consensus_by_flavor(flav);
+  const char *flavor = networkstatus_get_flavor_name(flav);
+  if (! we_want_to_fetch_flavor(get_options(), flav))
+    return;
+
+  if (c && c->valid_after <= now && now <= c->valid_until) {
+    long dl_interval;
+    long interval = c->fresh_until - c->valid_after;
+    long min_sec_before_caching = CONSENSUS_MIN_SECONDS_BEFORE_CACHING;
+    time_t start;
+
+    if (min_sec_before_caching > interval/16) {
+      /* Usually we allow 2-minutes slop factor in case clocks get
+         desynchronized a little.  If we're on a private network with
+         a crazy-fast voting interval, though, 2 minutes may be too
+         much. */
+      min_sec_before_caching = interval/16;
+      /* make sure we always delay by at least a second before caching */
+      if (min_sec_before_caching == 0) {
+        min_sec_before_caching = 1;
+      }
+    }
+
+    if (directory_fetches_dir_info_early(options)) {
+      /* We want to cache the next one at some point after this one
+       * is no longer fresh... */
+      start = (time_t)(c->fresh_until + min_sec_before_caching);
+      /* Some clients may need the consensus sooner than others. */
+      if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) {
+        dl_interval = 60;
+        if (min_sec_before_caching + dl_interval > interval)
+          dl_interval = interval/2;
+      } else {
+        /* But only in the first half-interval after that. */
+        dl_interval = interval/2;
+      }
+    } else {
+      /* We're an ordinary client, a bridge, or a hidden service.
+       * Give all the caches enough time to download the consensus. */
+      start = (time_t)(c->fresh_until + (interval*3)/4);
+      /* But download the next one well before this one is expired. */
+      dl_interval = ((c->valid_until - start) * 7 )/ 8;
+
+      /* If we're a bridge user, make use of the numbers we just computed
+       * to choose the rest of the interval *after* them. */
+      if (directory_fetches_dir_info_later(options)) {
+        /* Give all the *clients* enough time to download the consensus. */
+        start = (time_t)(start + dl_interval + min_sec_before_caching);
+        /* But try to get it before ours actually expires. */
+        dl_interval = (c->valid_until - start) - min_sec_before_caching;
+      }
+    }
+    /* catch low dl_interval in crazy-fast networks */
+    if (dl_interval < 1)
+      dl_interval = 1;
+    /* catch late start in crazy-fast networks */
+    if (start+dl_interval >= c->valid_until)
+      start = c->valid_until - dl_interval - 1;
+    log_debug(LD_DIR,
+              "fresh_until: %ld start: %ld "
+              "dl_interval: %ld valid_until: %ld ",
+              (long)c->fresh_until, (long)start, dl_interval,
+              (long)c->valid_until);
+    /* We must not try to replace c while it's still fresh: */
+    tor_assert(c->fresh_until < start);
+    /* We must download the next one before c is invalid: */
+    tor_assert(start+dl_interval < c->valid_until);
+    time_to_download_next_consensus[flav] =
+      start + crypto_rand_int((int)dl_interval);
+    {
+      char tbuf1[ISO_TIME_LEN+1];
+      char tbuf2[ISO_TIME_LEN+1];
+      char tbuf3[ISO_TIME_LEN+1];
+      format_local_iso_time(tbuf1, c->fresh_until);
+      format_local_iso_time(tbuf2, c->valid_until);
+      format_local_iso_time(tbuf3, time_to_download_next_consensus[flav]);
+      log_info(LD_DIR, "Live %s consensus %s the most recent until %s and "
+               "will expire at %s; fetching the next one at %s.",
+               flavor, (c->fresh_until > now) ? "will be" : "was",
+               tbuf1, tbuf2, tbuf3);
+    }
+  } else {
+    time_to_download_next_consensus[flav] = now;
+    log_info(LD_DIR, "No live %s consensus; we should fetch one immediately.",
+             flavor);
+  }
+}
+
+/** Update the time at which we'll consider replacing the current
+ * consensus of flavor 'flavor' */
+void
+update_consensus_networkstatus_fetch_time(time_t now)
+{
+  int i;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    if (we_want_to_fetch_flavor(get_options(), i))
+      update_consensus_networkstatus_fetch_time_impl(now, i);
+  }
+}
+
+/** Return 1 if there's a reason we shouldn't try any directory
+ * fetches yet (e.g. we demand bridges and none are yet known).
+ * Else return 0.
+
+ * If we return 1 and <b>msg_out</b> is provided, set <b>msg_out</b>
+ * to an explanation of why directory fetches are delayed. (If we
+ * return 0, we set msg_out to NULL.)
+ */
+int
+should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
+{
+  if (msg_out) {
+    *msg_out = NULL;
+  }
+
+  if (options->DisableNetwork) {
+    if (msg_out) {
+      *msg_out = "DisableNetwork is set.";
+    }
+    log_info(LD_DIR, "Delaying dir fetches (DisableNetwork is set)");
+    return 1;
+  }
+
+  if (we_are_hibernating()) {
+    if (msg_out) {
+      *msg_out = "We are hibernating or shutting down.";
+    }
+    log_info(LD_DIR, "Delaying dir fetches (Hibernating or shutting down)");
+    return 1;
+  }
+
+  if (options->UseBridges) {
+    /* If we know that none of our bridges can possibly work, avoid fetching
+     * directory documents. But if some of them might work, try again. */
+    if (num_bridges_usable(1) == 0) {
+      if (msg_out) {
+        *msg_out = "No running bridges";
+      }
+      log_info(LD_DIR, "Delaying dir fetches (no running bridges known)");
+      return 1;
+    }
+
+    if (pt_proxies_configuration_pending()) {
+      if (msg_out) {
+        *msg_out = "Pluggable transport proxies still configuring";
+      }
+      log_info(LD_DIR, "Delaying dir fetches (pt proxies still configuring)");
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+/** Launch requests for networkstatus documents as appropriate. This is called
+ * when we retry all the connections on a SIGHUP and periodically by a Periodic
+ * event which checks whether we want to download any networkstatus documents.
+ */
+void
+update_networkstatus_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  /** Launch a consensus download request, we will wait for the consensus to
+   * download and when it completes we will launch a certificate download
+   * request. */
+  update_consensus_networkstatus_downloads(now);
+}
+
+/** Launch requests as appropriate for missing directory authority
+ * certificates. */
+void
+update_certificate_downloads(time_t now)
+{
+  int i;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    if (consensus_waiting_for_certs[i].consensus)
+      authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus,
+                                    now, NULL);
+  }
+
+  if (current_ns_consensus)
+    authority_certs_fetch_missing(current_ns_consensus, now, NULL);
+  if (current_md_consensus)
+    authority_certs_fetch_missing(current_md_consensus, now, NULL);
+}
+
+/** Return 1 if we have a consensus but we don't have enough certificates
+ * to start using it yet. */
+int
+consensus_is_waiting_for_certs(void)
+{
+  return consensus_waiting_for_certs[usable_consensus_flavor()].consensus
+    ? 1 : 0;
+}
+
+/** Look up the currently active (depending on bootstrap status) download
+ * status for this consensus flavor and return a pointer to it.
+ */
+MOCK_IMPL(download_status_t *,
+networkstatus_get_dl_status_by_flavor,(consensus_flavor_t flavor))
+{
+  download_status_t *dl = NULL;
+  const int we_are_bootstrapping =
+    networkstatus_consensus_is_bootstrapping(time(NULL));
+
+  if ((int)flavor <= N_CONSENSUS_FLAVORS) {
+    dl = &((we_are_bootstrapping ?
+           consensus_bootstrap_dl_status : consensus_dl_status)[flavor]);
+  }
+
+  return dl;
+}
+
+/** Look up the bootstrap download status for this consensus flavor
+ * and return a pointer to it. */
+MOCK_IMPL(download_status_t *,
+networkstatus_get_dl_status_by_flavor_bootstrap,(consensus_flavor_t flavor))
+{
+  download_status_t *dl = NULL;
+
+  if ((int)flavor <= N_CONSENSUS_FLAVORS) {
+    dl = &(consensus_bootstrap_dl_status[flavor]);
+  }
+
+  return dl;
+}
+
+/** Look up the running (non-bootstrap) download status for this consensus
+ * flavor and return a pointer to it. */
+MOCK_IMPL(download_status_t *,
+networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor))
+{
+  download_status_t *dl = NULL;
+
+  if ((int)flavor <= N_CONSENSUS_FLAVORS) {
+    dl = &(consensus_dl_status[flavor]);
+  }
+
+  return dl;
+}
+
+/** Return the most recent consensus that we have downloaded, or NULL if we
+ * don't have one. */
+MOCK_IMPL(networkstatus_t *,
+networkstatus_get_latest_consensus,(void))
+{
+  if (we_use_microdescriptors_for_circuits(get_options()))
+    return current_md_consensus;
+  else
+    return current_ns_consensus;
+}
+
+/** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL
+ * if we don't have one. */
+MOCK_IMPL(networkstatus_t *,
+networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
+{
+  if (f == FLAV_NS)
+    return current_ns_consensus;
+  else if (f == FLAV_MICRODESC)
+    return current_md_consensus;
+  else {
+    tor_assert(0);
+    return NULL;
+  }
+}
+
+/** Return the most recent consensus that we have downloaded, or NULL if it is
+ * no longer live. */
+MOCK_IMPL(networkstatus_t *,
+networkstatus_get_live_consensus,(time_t now))
+{
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (ns && networkstatus_is_live(ns, now))
+    return ns;
+  else
+    return NULL;
+}
+
+/** Given a consensus in <b>ns</b>, return true iff currently live and
+ *  unexpired. */
+int
+networkstatus_is_live(const networkstatus_t *ns, time_t now)
+{
+  return (ns->valid_after <= now && now <= ns->valid_until);
+}
+
+/** Determine if <b>consensus</b> is valid or expired recently enough that
+ * we can still use it.
+ *
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ */
+int
+networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                        time_t now)
+{
+  if (BUG(!consensus))
+    return 0;
+
+  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+                                                      now);
+}
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_until
+ * time rather than an entire consensus. */
+int
+networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                             time_t now)
+{
+#define REASONABLY_LIVE_TIME (24*60*60)
+  return (now <= valid_until + REASONABLY_LIVE_TIME);
+}
+
+/* XXXX remove this in favor of get_live_consensus. But actually,
+ * leave something like it for bridge users, who need to not totally
+ * lose if they spend a while fetching a new consensus. */
+/** As networkstatus_get_live_consensus(), but is way more tolerant of expired
+ * consensuses. */
+networkstatus_t *
+networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
+{
+  networkstatus_t *consensus =
+    networkstatus_get_latest_consensus_by_flavor(flavor);
+  if (consensus &&
+      consensus->valid_after <= now &&
+      networkstatus_consensus_reasonably_live(consensus, now))
+    return consensus;
+  else
+    return NULL;
+}
+
+/** Check if we need to download a consensus during tor's bootstrap phase.
+ * If we have no consensus, or our consensus is unusably old, return 1.
+ * As soon as we have received a consensus, return 0, even if we don't have
+ * enough certificates to validate it.
+ * If a fallback directory gives us a consensus we can never get certs for,
+ * check_consensus_waiting_for_certs() will wait 20 minutes before failing
+ * the cert downloads. After that, a new consensus will be fetched from a
+ * randomly chosen fallback. */
+MOCK_IMPL(int,
+networkstatus_consensus_is_bootstrapping,(time_t now))
+{
+  /* If we have a validated, reasonably live consensus, we're not
+   * bootstrapping a consensus at all. */
+  if (networkstatus_get_reasonably_live_consensus(
+                                                now,
+                                                usable_consensus_flavor())) {
+    return 0;
+  }
+
+  /* If we have a consensus, but we're waiting for certificates,
+   * we're not waiting for a consensus download while bootstrapping. */
+  if (consensus_is_waiting_for_certs()) {
+    return 0;
+  }
+
+  /* If we have no consensus, or our consensus is very old, we are
+   * bootstrapping, and we need to download a consensus. */
+  return 1;
+}
+
+/** Check if we can use multiple directories for a consensus download.
+ * Only clients (including bridge relays, which act like clients) benefit
+ * from multiple simultaneous consensus downloads. */
+int
+networkstatus_consensus_can_use_multiple_directories(
+                                                  const or_options_t *options)
+{
+  /* If we are a client, bridge, bridge client, or hidden service */
+  return !public_server_mode(options);
+}
+
+/** Check if we can use fallback directory mirrors for a consensus download.
+ * If we have fallbacks and don't want to fetch from the authorities,
+ * we can use them. */
+MOCK_IMPL(int,
+networkstatus_consensus_can_use_extra_fallbacks,(const or_options_t *options))
+{
+  /* The list length comparisons are a quick way to check if we have any
+   * non-authority fallback directories. If we ever have any authorities that
+   * aren't fallback directories, we will need to change this code. */
+  tor_assert(smartlist_len(router_get_fallback_dir_servers())
+             >= smartlist_len(router_get_trusted_dir_servers()));
+  /* If we don't fetch from the authorities, and we have additional mirrors,
+   * we can use them. */
+  return (!directory_fetches_from_authorities(options)
+          && (smartlist_len(router_get_fallback_dir_servers())
+              > smartlist_len(router_get_trusted_dir_servers())));
+}
+
+/* Is there a consensus fetch for flavor <b>resource</b> that's far
+ * enough along to be attached to a circuit? */
+int
+networkstatus_consensus_is_already_downloading(const char *resource)
+{
+  int answer = 0;
+
+  /* First, get a list of all the dir conns that are fetching a consensus,
+   * fetching *this* consensus, and are in state "reading" (meaning they
+   * have already flushed their request onto the socks connection). */
+  smartlist_t *fetching_conns =
+    connection_dir_list_by_purpose_resource_and_state(
+      DIR_PURPOSE_FETCH_CONSENSUS, resource, DIR_CONN_STATE_CLIENT_READING);
+
+  /* Then, walk through each conn, to see if its linked socks connection
+   * is in an attached state. We have to check this separately, since with
+   * the optimistic data feature, fetches can send their request to the
+   * socks connection and go into state 'reading', even before they're
+   * attached to any circuit. */
+  SMARTLIST_FOREACH_BEGIN(fetching_conns, dir_connection_t *, dirconn) {
+    /* Do any of these other dir conns have a linked socks conn that is
+     * attached to a circuit already? */
+    connection_t *base = TO_CONN(dirconn);
+    if (base->linked_conn &&
+        base->linked_conn->type == CONN_TYPE_AP &&
+        !AP_CONN_STATE_IS_UNATTACHED(base->linked_conn->state)) {
+      answer = 1;
+      break; /* stop looping, because we know the answer will be yes */
+    }
+  } SMARTLIST_FOREACH_END(dirconn);
+  smartlist_free(fetching_conns);
+
+  return answer;
+}
+
+/* Does the current, reasonably live consensus have IPv6 addresses?
+ * Returns 1 if there is a reasonably live consensus and its consensus method
+ * includes IPv6 addresses in the consensus.
+ * Otherwise, if there is no consensus, or the method does not include IPv6
+ * addresses, returns 0. */
+int
+networkstatus_consensus_has_ipv6(const or_options_t* options)
+{
+  const networkstatus_t *cons = networkstatus_get_reasonably_live_consensus(
+                                                    approx_time(),
+                                                    usable_consensus_flavor());
+
+  /* If we have no consensus, we have no IPv6 in it */
+  if (!cons) {
+    return 0;
+  }
+
+  /* Different flavours of consensus gained IPv6 at different times */
+  if (we_use_microdescriptors_for_circuits(options)) {
+    return
+       cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+  } else {
+    return 1;
+  }
+}
+
+/** Given two router status entries for the same router identity, return 1 if
+ * if the contents have changed between them. Otherwise, return 0. */
+static int
+routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b)
+{
+  tor_assert(tor_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN));
+
+  return strcmp(a->nickname, b->nickname) ||
+         fast_memneq(a->descriptor_digest, b->descriptor_digest, DIGEST_LEN) ||
+         a->addr != b->addr ||
+         a->or_port != b->or_port ||
+         a->dir_port != b->dir_port ||
+         a->is_authority != b->is_authority ||
+         a->is_exit != b->is_exit ||
+         a->is_stable != b->is_stable ||
+         a->is_fast != b->is_fast ||
+         a->is_flagged_running != b->is_flagged_running ||
+         a->is_named != b->is_named ||
+         a->is_unnamed != b->is_unnamed ||
+         a->is_valid != b->is_valid ||
+         a->is_possible_guard != b->is_possible_guard ||
+         a->is_bad_exit != b->is_bad_exit ||
+         a->is_hs_dir != b->is_hs_dir;
+  // XXXX this function needs a huge refactoring; it has gotten out
+  // XXXX of sync with routerstatus_t, and it will do so again.
+}
+
+/** Notify controllers of any router status entries that changed between
+ * <b>old_c</b> and <b>new_c</b>. */
+static void
+notify_control_networkstatus_changed(const networkstatus_t *old_c,
+                                     const networkstatus_t *new_c)
+{
+  smartlist_t *changed;
+  if (old_c == new_c)
+    return;
+
+  /* tell the controller exactly which relays are still listed, as well
+   * as what they're listed as */
+  control_event_newconsensus(new_c);
+
+  if (!control_event_is_interesting(EVENT_NS))
+    return;
+
+  if (!old_c) {
+    control_event_networkstatus_changed(new_c->routerstatus_list);
+    return;
+  }
+  changed = smartlist_new();
+
+  SMARTLIST_FOREACH_JOIN(
+                     old_c->routerstatus_list, const routerstatus_t *, rs_old,
+                     new_c->routerstatus_list, const routerstatus_t *, rs_new,
+                     tor_memcmp(rs_old->identity_digest,
+                            rs_new->identity_digest, DIGEST_LEN),
+                     smartlist_add(changed, (void*) rs_new)) {
+    if (routerstatus_has_changed(rs_old, rs_new))
+      smartlist_add(changed, (void*)rs_new);
+  } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new);
+
+  control_event_networkstatus_changed(changed);
+  smartlist_free(changed);
+}
+
+/* Called before the consensus changes from old_c to new_c. */
+static void
+notify_before_networkstatus_changes(const networkstatus_t *old_c,
+                                    const networkstatus_t *new_c)
+{
+  notify_control_networkstatus_changed(old_c, new_c);
+  dos_consensus_has_changed(new_c);
+  relay_consensus_has_changed(new_c);
+}
+
+/* Called after a new consensus has been put in the global state. It is safe
+ * to use the consensus getters in this function. */
+static void
+notify_after_networkstatus_changes(void)
+{
+  scheduler_notify_networkstatus_changed();
+}
+
+/** Copy all the ancillary information (like router download status and so on)
+ * from <b>old_c</b> to <b>new_c</b>. */
+static void
+networkstatus_copy_old_consensus_info(networkstatus_t *new_c,
+                                      const networkstatus_t *old_c)
+{
+  if (old_c == new_c)
+    return;
+  if (!old_c || !smartlist_len(old_c->routerstatus_list))
+    return;
+
+  SMARTLIST_FOREACH_JOIN(old_c->routerstatus_list, routerstatus_t *, rs_old,
+                         new_c->routerstatus_list, routerstatus_t *, rs_new,
+                         tor_memcmp(rs_old->identity_digest,
+                                rs_new->identity_digest, DIGEST_LEN),
+                         STMT_NIL) {
+    /* Okay, so we're looking at the same identity. */
+    rs_new->last_dir_503_at = rs_old->last_dir_503_at;
+
+    if (tor_memeq(rs_old->descriptor_digest, rs_new->descriptor_digest,
+                  DIGEST256_LEN)) {
+      /* And the same descriptor too! */
+      memcpy(&rs_new->dl_status, &rs_old->dl_status,sizeof(download_status_t));
+    }
+  } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new);
+}
+
+#ifdef TOR_UNIT_TESTS
+/**Accept a <b>flavor</b> consensus <b>c</b> without any additional
+ * validation. This is exclusively for unit tests.
+ * We copy any ancillary information from a pre-existing consensus
+ * and then free the current one and replace it with the newly
+ * provided instance. Returns -1 on unrecognized flavor, 0 otherwise.
+ */
+int
+networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
+                                            const char *flavor)
+{
+  int flav = networkstatus_parse_flavor_name(flavor);
+  switch (flav) {
+    case FLAV_NS:
+      if (current_ns_consensus) {
+        networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+        networkstatus_vote_free(current_ns_consensus);
+      }
+      current_ns_consensus = c;
+      break;
+    case FLAV_MICRODESC:
+      if (current_md_consensus) {
+        networkstatus_copy_old_consensus_info(c, current_md_consensus);
+        networkstatus_vote_free(current_md_consensus);
+      }
+      current_md_consensus = c;
+      break;
+  }
+  return current_md_consensus ? 0 : -1;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Helper for handle_missing_protocol_warning: handles either the
+ * client case (if <b>is_client</b> is set) or the server case otherwise.
+ */
+static void
+handle_missing_protocol_warning_impl(const networkstatus_t *c,
+                                     int is_client)
+{
+  char *protocol_warning = NULL;
+
+  int should_exit = networkstatus_check_required_protocols(c,
+                                                   is_client,
+                                                   &protocol_warning);
+  if (protocol_warning) {
+    tor_log(should_exit ? LOG_ERR : LOG_WARN,
+            LD_GENERAL,
+            "%s", protocol_warning);
+  }
+  if (should_exit) {
+    tor_assert_nonfatal(protocol_warning);
+  }
+  tor_free(protocol_warning);
+  if (should_exit)
+    exit(1); // XXXX bad exit: should return from main.
+}
+
+/** Called when we have received a networkstatus <b>c</b>. If there are
+ * any _required_ protocols we are missing, log an error and exit
+ * immediately. If there are any _recommended_ protocols we are missing,
+ * warn. */
+static void
+handle_missing_protocol_warning(const networkstatus_t *c,
+                                const or_options_t *options)
+{
+  const int is_server = server_mode(options);
+  const int is_client = options_any_client_port_set(options) || !is_server;
+
+  if (is_server)
+    handle_missing_protocol_warning_impl(c, 0);
+  if (is_client)
+    handle_missing_protocol_warning_impl(c, 1);
+}
+
+/**
+ * Check whether we received a consensus that appears to be coming
+ * from the future.  Because we implicitly trust the directory
+ * authorities' idea of the current time, we produce a warning if we
+ * get an early consensus.
+ *
+ * If we got a consensus that is time stamped far in the past, that
+ * could simply have come from a stale cache.  Possible ways to get a
+ * consensus from the future can include:
+ *
+ * - enough directory authorities have wrong clocks
+ * - directory authorities collude to produce misleading time stamps
+ * - our own clock is wrong (this is by far the most likely)
+ *
+ * We neglect highly improbable scenarios that involve actual time
+ * travel.
+ */
+STATIC void
+warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                     time_t now)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  char dbuf[64];
+  long delta = now - c->valid_after;
+  char *flavormsg = NULL;
+
+/** If a consensus appears more than this many seconds before it could
+ * possibly be a sufficiently-signed consensus, declare that our clock
+ * is skewed. */
+#define EARLY_CONSENSUS_NOTICE_SKEW 60
+
+  /* We assume that if a majority of dirauths have accurate clocks,
+   * the earliest that a dirauth with a skewed clock could possibly
+   * publish a sufficiently-signed consensus is (valid_after -
+   * dist_seconds).  Before that time, the skewed dirauth would be
+   * unable to obtain enough authority signatures for the consensus to
+   * be valid. */
+  if (now >= c->valid_after - c->dist_seconds - EARLY_CONSENSUS_NOTICE_SKEW)
+    return;
+
+  format_iso_time(tbuf, c->valid_after);
+  format_time_interval(dbuf, sizeof(dbuf), delta);
+  log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
+           "consensus network status document (%s UTC).  Tor needs an "
+           "accurate clock to work correctly. Please check your time and "
+           "date settings!", dbuf, tbuf);
+  tor_asprintf(&flavormsg, "%s flavor consensus", flavor);
+  clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS");
+  tor_free(flavormsg);
+}
+
+/** Try to replace the current cached v3 networkstatus with the one in
+ * <b>consensus</b>.  If we don't have enough certificates to validate it,
+ * store it in consensus_waiting_for_certs and launch a certificate fetch.
+ *
+ * If flags & NSSET_FROM_CACHE, this networkstatus has come from the disk
+ * cache.  If flags & NSSET_WAS_WAITING_FOR_CERTS, this networkstatus was
+ * already received, but we were waiting for certificates on it.  If flags &
+ * NSSET_DONT_DOWNLOAD_CERTS, do not launch certificate downloads as needed.
+ * If flags & NSSET_ACCEPT_OBSOLETE, then we should be willing to take this
+ * consensus, even if it comes from many days in the past.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved a consensus or certificates from, so try
+ * it first to fetch any missing certificates.
+ *
+ * Return 0 on success, <0 on failure.  On failure, caller should increment
+ * the failure count as appropriate.
+ *
+ * We return -1 for mild failures that don't need to be reported to the
+ * user, and -2 for more serious problems.
+ */
+int
+networkstatus_set_current_consensus(const char *consensus,
+                                    const char *flavor,
+                                    unsigned flags,
+                                    const char *source_dir)
+{
+  networkstatus_t *c=NULL;
+  int r, result = -1;
+  time_t now = approx_time();
+  const or_options_t *options = get_options();
+  char *unverified_fname = NULL, *consensus_fname = NULL;
+  int flav = networkstatus_parse_flavor_name(flavor);
+  const unsigned from_cache = flags & NSSET_FROM_CACHE;
+  const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS;
+  const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS);
+  const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE;
+  const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR;
+  const common_digests_t *current_digests = NULL;
+  consensus_waiting_for_certs_t *waiting = NULL;
+  time_t current_valid_after = 0;
+  int free_consensus = 1; /* Free 'c' at the end of the function */
+  int checked_protocols_already = 0;
+
+  if (flav < 0) {
+    /* XXXX we don't handle unrecognized flavors yet. */
+    log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor);
+    return -2;
+  }
+
+  /* Make sure it's parseable. */
+  c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS);
+  if (!c) {
+    log_warn(LD_DIR, "Unable to parse networkstatus consensus");
+    result = -2;
+    goto done;
+  }
+
+  if (from_cache && !was_waiting_for_certs) {
+    /* We previously stored this; check _now_ to make sure that version-kills
+     * really work. This happens even before we check signatures: we did so
+     * before when we stored this to disk. This does mean an attacker who can
+     * write to the datadir can make us not start: such an attacker could
+     * already harm us by replacing our guards, which would be worse. */
+    checked_protocols_already = 1;
+    handle_missing_protocol_warning(c, options);
+  }
+
+  if ((int)c->flavor != flav) {
+    /* This wasn't the flavor we thought we were getting. */
+    if (require_flavor) {
+      log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)",
+               networkstatus_get_flavor_name(c->flavor), flavor);
+      goto done;
+    }
+    flav = c->flavor;
+    flavor = networkstatus_get_flavor_name(flav);
+  }
+
+  if (flav != usable_consensus_flavor() &&
+      !we_want_to_fetch_flavor(options, flav)) {
+    /* This consensus is totally boring to us: we won't use it, we didn't want
+     * it, and we won't serve it.  Drop it. */
+    goto done;
+  }
+
+  if (from_cache && !accept_obsolete &&
+      c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) {
+    log_info(LD_DIR, "Loaded an expired consensus. Discarding.");
+    goto done;
+  }
+
+  if (!strcmp(flavor, "ns")) {
+    consensus_fname = get_cachedir_fname("cached-consensus");
+    unverified_fname = get_cachedir_fname("unverified-consensus");
+    if (current_ns_consensus) {
+      current_digests = &current_ns_consensus->digests;
+      current_valid_after = current_ns_consensus->valid_after;
+    }
+  } else if (!strcmp(flavor, "microdesc")) {
+    consensus_fname = get_cachedir_fname("cached-microdesc-consensus");
+    unverified_fname = get_cachedir_fname("unverified-microdesc-consensus");
+    if (current_md_consensus) {
+      current_digests = &current_md_consensus->digests;
+      current_valid_after = current_md_consensus->valid_after;
+    }
+  } else {
+    tor_assert_nonfatal_unreached();
+    result = -2;
+    goto done;
+  }
+
+  if (current_digests &&
+      tor_memeq(&c->digests, current_digests, sizeof(c->digests))) {
+    /* We already have this one. That's a failure. */
+    log_info(LD_DIR, "Got a %s consensus we already have", flavor);
+    goto done;
+  }
+
+  if (current_valid_after && c->valid_after <= current_valid_after) {
+    /* We have a newer one.  There's no point in accepting this one,
+     * even if it's great. */
+    log_info(LD_DIR, "Got a %s consensus at least as old as the one we have",
+             flavor);
+    goto done;
+  }
+
+  /* Make sure it's signed enough. */
+  if ((r=networkstatus_check_consensus_signature(c, 1))<0) {
+    if (r == -1) {
+      /* Okay, so it _might_ be signed enough if we get more certificates. */
+      if (!was_waiting_for_certs) {
+        log_info(LD_DIR,
+                 "Not enough certificates to check networkstatus consensus");
+      }
+      if (!current_valid_after ||
+          c->valid_after > current_valid_after) {
+        waiting = &consensus_waiting_for_certs[flav];
+        networkstatus_vote_free(waiting->consensus);
+        tor_free(waiting->body);
+        waiting->consensus = c;
+        free_consensus = 0;
+        waiting->body = tor_strdup(consensus);
+        waiting->set_at = now;
+        waiting->dl_failed = 0;
+        if (!from_cache) {
+          write_str_to_file(unverified_fname, consensus, 0);
+        }
+        if (dl_certs)
+          authority_certs_fetch_missing(c, now, source_dir);
+        /* This case is not a success or a failure until we get the certs
+         * or fail to get the certs. */
+        result = 0;
+      } else {
+        /* Even if we had enough signatures, we'd never use this as the
+         * latest consensus. */
+        if (was_waiting_for_certs && from_cache)
+          if (unlink(unverified_fname) != 0) {
+            log_warn(LD_FS,
+                     "Failed to unlink %s: %s",
+                     unverified_fname, strerror(errno));
+          }
+      }
+      goto done;
+    } else {
+      /* This can never be signed enough:  Kill it. */
+      if (!was_waiting_for_certs) {
+        log_warn(LD_DIR, "Not enough good signatures on networkstatus "
+                 "consensus");
+        result = -2;
+      }
+      if (was_waiting_for_certs && (r < -1) && from_cache) {
+        if (unlink(unverified_fname) != 0) {
+            log_warn(LD_FS,
+                     "Failed to unlink %s: %s",
+                     unverified_fname, strerror(errno));
+        }
+      }
+      goto done;
+    }
+  }
+
+  /* Signatures from the consensus are verified */
+  if (from_cache && was_waiting_for_certs) {
+    /* We check if the consensus is loaded from disk cache and that it
+     * it is an unverified consensus. If it is unverified, rename it to
+     * cached-*-consensus since it has been verified. */
+    log_info(LD_DIR, "Unverified consensus signatures verified.");
+    tor_rename(unverified_fname, consensus_fname);
+  }
+
+  if (!from_cache && flav == usable_consensus_flavor())
+    control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED");
+
+  if (!checked_protocols_already) {
+    handle_missing_protocol_warning(c, options);
+  }
+
+  /* Are we missing any certificates at all? */
+  if (r != 1 && dl_certs)
+    authority_certs_fetch_missing(c, now, source_dir);
+
+  const int is_usable_flavor = flav == usable_consensus_flavor();
+
+  /* Before we switch to the new consensus, notify that we are about to change
+   * it using the old consensus and the new one. */
+  if (is_usable_flavor) {
+    notify_before_networkstatus_changes(networkstatus_get_latest_consensus(),
+                                        c);
+  }
+  if (flav == FLAV_NS) {
+    if (current_ns_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+      networkstatus_vote_free(current_ns_consensus);
+      /* Defensive programming : we should set current_ns_consensus very soon
+       * but we're about to call some stuff in the meantime, and leaving this
+       * dangling pointer around has proven to be trouble. */
+      current_ns_consensus = NULL;
+    }
+    current_ns_consensus = c;
+    free_consensus = 0; /* avoid free */
+  } else if (flav == FLAV_MICRODESC) {
+    if (current_md_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_md_consensus);
+      networkstatus_vote_free(current_md_consensus);
+      /* more defensive programming */
+      current_md_consensus = NULL;
+    }
+    current_md_consensus = c;
+    free_consensus = 0; /* avoid free */
+  }
+
+  waiting = &consensus_waiting_for_certs[flav];
+  if (waiting->consensus &&
+      waiting->consensus->valid_after <= c->valid_after) {
+    networkstatus_vote_free(waiting->consensus);
+    waiting->consensus = NULL;
+    if (consensus != waiting->body)
+      tor_free(waiting->body);
+    else
+      waiting->body = NULL;
+    waiting->set_at = 0;
+    waiting->dl_failed = 0;
+    if (unlink(unverified_fname) != 0) {
+      log_warn(LD_FS,
+               "Failed to unlink %s: %s",
+               unverified_fname, strerror(errno));
+    }
+  }
+
+  if (is_usable_flavor) {
+    /* Notify that we just changed the consensus so the current global value
+     * can be looked at. */
+    notify_after_networkstatus_changes();
+
+    /* The "current" consensus has just been set and it is a usable flavor so
+     * the first thing we need to do is recalculate the voting schedule static
+     * object so we can use the timings in there needed by some subsystems
+     * such as hidden service and shared random. */
+    voting_schedule_recalculate_timing(options, now);
+    reschedule_dirvote(options);
+
+    nodelist_set_consensus(c);
+
+    /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
+    update_consensus_networkstatus_fetch_time(now);
+
+    /* Change the cell EWMA settings */
+    cmux_ewma_set_options(options, c);
+
+    /* XXXX this call might be unnecessary here: can changing the
+     * current consensus really alter our view of any OR's rate limits? */
+    connection_or_update_token_buckets(get_connection_array(), options);
+
+    circuit_build_times_new_consensus_params(
+                               get_circuit_build_times_mutable(), c);
+    channelpadding_new_consensus_params(c);
+  }
+
+  /* Reset the failure count only if this consensus is actually valid. */
+  if (c->valid_after <= now && now <= c->valid_until) {
+    download_status_reset(&consensus_dl_status[flav]);
+  } else {
+    if (!from_cache)
+      download_status_failed(&consensus_dl_status[flav], 0);
+  }
+
+  if (we_want_to_fetch_flavor(options, flav)) {
+    dirserv_set_cached_consensus_networkstatus(consensus,
+                                               flavor,
+                                               &c->digests,
+                                               c->digest_sha3_as_signed,
+                                               c->valid_after);
+    if (dir_server_mode(get_options())) {
+      consdiffmgr_add_consensus(consensus, c);
+    }
+  }
+
+  if (!from_cache) {
+    write_str_to_file(consensus_fname, consensus, 0);
+  }
+
+  warn_early_consensus(c, flavor, now);
+
+  /* We got a new consesus. Reset our md fetch fail cache */
+  microdesc_reset_outdated_dirservers_list();
+
+  router_dir_info_changed();
+
+  result = 0;
+ done:
+  if (free_consensus)
+    networkstatus_vote_free(c);
+  tor_free(consensus_fname);
+  tor_free(unverified_fname);
+  return result;
+}
+
+/** Called when we have gotten more certificates: see whether we can
+ * now verify a pending consensus.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved certificates from, so try it first to
+ * fetch any missing certificates.
+ */
+void
+networkstatus_note_certs_arrived(const char *source_dir)
+{
+  int i;
+  for (i=0; i<N_CONSENSUS_FLAVORS; ++i) {
+    const char *flavor_name = networkstatus_get_flavor_name(i);
+    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    if (!waiting->consensus)
+      continue;
+    if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
+      char *waiting_body = waiting->body;
+      if (!networkstatus_set_current_consensus(
+                                 waiting_body,
+                                 flavor_name,
+                                 NSSET_WAS_WAITING_FOR_CERTS,
+                                 source_dir)) {
+        tor_free(waiting_body);
+      }
+    }
+  }
+}
+
+/** If the network-status list has changed since the last time we called this
+ * function, update the status of every routerinfo from the network-status
+ * list. If <b>dir_version</b> is 2, it's a v2 networkstatus that changed.
+ * If <b>dir_version</b> is 3, it's a v3 consensus that changed.
+ */
+void
+routers_update_all_from_networkstatus(time_t now, int dir_version)
+{
+  routerlist_t *rl = router_get_routerlist();
+  networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,
+                                                                     FLAV_NS);
+
+  if (!consensus || dir_version < 3) /* nothing more we should do */
+    return;
+
+  /* calls router_dir_info_changed() when it's done -- more routers
+   * might be up or down now, which might affect whether there's enough
+   * directory info. */
+  routers_update_status_from_consensus_networkstatus(rl->routers, 0);
+
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri,
+                    ri->cache_info.routerlist_index = ri_sl_idx);
+  if (rl->old_routers)
+    signed_descs_update_status_from_consensus_networkstatus(rl->old_routers);
+
+  if (!have_warned_about_old_version) {
+    int is_server = server_mode(get_options());
+    version_status_t status;
+    const char *recommended = is_server ?
+      consensus->server_versions : consensus->client_versions;
+    status = tor_version_is_obsolete(VERSION, recommended);
+
+    if (status == VS_RECOMMENDED) {
+      log_info(LD_GENERAL, "The directory authorities say my version is ok.");
+    } else if (status == VS_EMPTY) {
+      log_info(LD_GENERAL,
+               "The directory authorities don't recommend any versions.");
+    } else if (status == VS_NEW || status == VS_NEW_IN_SERIES) {
+      if (!have_warned_about_new_version) {
+        log_notice(LD_GENERAL, "This version of Tor (%s) is newer than any "
+                   "recommended version%s, according to the directory "
+                   "authorities. Recommended versions are: %s",
+                   VERSION,
+                   status == VS_NEW_IN_SERIES ? " in its series" : "",
+                   recommended);
+        have_warned_about_new_version = 1;
+        control_event_general_status(LOG_WARN, "DANGEROUS_VERSION "
+                                     "CURRENT=%s REASON=%s RECOMMENDED=\"%s\"",
+                                     VERSION, "NEW", recommended);
+      }
+    } else {
+      log_warn(LD_GENERAL, "Please upgrade! "
+               "This version of Tor (%s) is %s, according to the directory "
+               "authorities. Recommended versions are: %s",
+               VERSION,
+               status == VS_OLD ? "obsolete" : "not recommended",
+               recommended);
+      have_warned_about_old_version = 1;
+      control_event_general_status(LOG_WARN, "DANGEROUS_VERSION "
+           "CURRENT=%s REASON=%s RECOMMENDED=\"%s\"",
+           VERSION, status == VS_OLD ? "OBSOLETE" : "UNRECOMMENDED",
+           recommended);
+    }
+  }
+}
+
+/** Given a list <b>routers</b> of routerinfo_t *, update each status field
+ * according to our current consensus networkstatus.  May re-order
+ * <b>routers</b>. */
+void
+routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
+                                                   int reset_failures)
+{
+  const or_options_t *options = get_options();
+  int authdir = authdir_mode_v3(options);
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (!ns || !smartlist_len(ns->routerstatus_list))
+    return;
+
+  routers_sort_by_identity(routers);
+
+  SMARTLIST_FOREACH_JOIN(ns->routerstatus_list, routerstatus_t *, rs,
+                         routers, routerinfo_t *, router,
+                         tor_memcmp(rs->identity_digest,
+                               router->cache_info.identity_digest, DIGEST_LEN),
+  {
+  }) {
+    /* Is it the same descriptor, or only the same identity? */
+    if (tor_memeq(router->cache_info.signed_descriptor_digest,
+                rs->descriptor_digest, DIGEST_LEN)) {
+      if (ns->valid_until > router->cache_info.last_listed_as_valid_until)
+        router->cache_info.last_listed_as_valid_until = ns->valid_until;
+    }
+
+    if (authdir) {
+      /* If we _are_ an authority, we should check whether this router
+       * is one that will cause us to need a reachability test. */
+      routerinfo_t *old_router =
+        router_get_mutable_by_digest(router->cache_info.identity_digest);
+      if (old_router != router) {
+        router->needs_retest_if_added =
+          dirserv_should_launch_reachability_test(router, old_router);
+      }
+    }
+    if (reset_failures) {
+      download_status_reset(&rs->dl_status);
+    }
+  } SMARTLIST_FOREACH_JOIN_END(rs, router);
+
+  router_dir_info_changed();
+}
+
+/** Given a list of signed_descriptor_t, update their fields (mainly, when
+ * they were last listed) from the most recent consensus. */
+void
+signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
+{
+  networkstatus_t *ns = current_ns_consensus;
+  if (!ns)
+    return;
+
+  if (!ns->desc_digest_map) {
+    char dummy[DIGEST_LEN];
+    /* instantiates the digest map. */
+    memset(dummy, 0, sizeof(dummy));
+    router_get_consensus_status_by_descriptor_digest(ns, dummy);
+  }
+  SMARTLIST_FOREACH(descs, signed_descriptor_t *, d,
+  {
+    const routerstatus_t *rs = digestmap_get(ns->desc_digest_map,
+                                       d->signed_descriptor_digest);
+    if (rs) {
+      if (ns->valid_until > d->last_listed_as_valid_until)
+        d->last_listed_as_valid_until = ns->valid_until;
+    }
+  });
+}
+
+/** Generate networkstatus lines for a single routerstatus_t object, and
+ * return the result in a newly allocated string.  Used only by controller
+ * interface (for now.) */
+char *
+networkstatus_getinfo_helper_single(const routerstatus_t *rs)
+{
+  return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT,
+                                   ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
+                                   NULL);
+}
+
+/** Alloc and return a string describing routerstatuses for the most
+ * recent info of each router we know about that is of purpose
+ * <b>purpose_string</b>. Return NULL if unrecognized purpose.
+ *
+ * Right now this function is oriented toward listing bridges (you
+ * shouldn't use this for general-purpose routers, since those
+ * should be listed from the consensus, not from the routers list). */
+char *
+networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
+{
+  const time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  char *answer;
+  routerlist_t *rl = router_get_routerlist();
+  smartlist_t *statuses;
+  const uint8_t purpose = router_purpose_from_string(purpose_string);
+  routerstatus_t rs;
+  const int bridge_auth = authdir_mode_bridge(get_options());
+
+  if (purpose == ROUTER_PURPOSE_UNKNOWN) {
+    log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
+             purpose_string);
+    return NULL;
+  }
+
+  statuses = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+    node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+    if (!node)
+      continue;
+    if (ri->cache_info.published_on < cutoff)
+      continue;
+    if (ri->purpose != purpose)
+      continue;
+    /* TODO: modifying the running flag in a getinfo is a bad idea */
+    if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      dirserv_set_router_is_running(ri, now);
+    /* then generate and write out status lines for each of them */
+    set_routerstatus_from_routerinfo(&rs, node, ri, now, 0);
+    smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs));
+  } SMARTLIST_FOREACH_END(ri);
+
+  answer = smartlist_join_strings(statuses, "", 0, NULL);
+  SMARTLIST_FOREACH(statuses, char *, cp, tor_free(cp));
+  smartlist_free(statuses);
+  return answer;
+}
+
+/** Write out router status entries for all our bridge descriptors. */
+void
+networkstatus_dump_bridge_status_to_file(time_t now)
+{
+  char *status = networkstatus_getinfo_by_purpose("bridge", now);
+  char *fname = NULL;
+  char *thresholds = NULL;
+  char *published_thresholds_and_status = NULL;
+  char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
+
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
+  format_iso_time(published, now);
+  dirserv_compute_bridge_flag_thresholds();
+  thresholds = dirserv_get_flag_thresholds_line();
+  tor_asprintf(&published_thresholds_and_status,
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
+  write_str_to_file(fname,published_thresholds_and_status,0);
+  tor_free(thresholds);
+  tor_free(published_thresholds_and_status);
+  tor_free(fname);
+  tor_free(status);
+  tor_free(fingerprint_line);
+}
+
+/* DOCDOC get_net_param_from_list */
+static int32_t
+get_net_param_from_list(smartlist_t *net_params, const char *param_name,
+                        int32_t default_val, int32_t min_val, int32_t max_val)
+{
+  int32_t res = default_val;
+  size_t name_len = strlen(param_name);
+
+  tor_assert(max_val > min_val);
+  tor_assert(min_val <= default_val);
+  tor_assert(max_val >= default_val);
+
+  SMARTLIST_FOREACH_BEGIN(net_params, const char *, p) {
+    if (!strcmpstart(p, param_name) && p[name_len] == '=') {
+      int ok=0;
+      long v = tor_parse_long(p+name_len+1, 10, INT32_MIN,
+                              INT32_MAX, &ok, NULL);
+      if (ok) {
+        res = (int32_t) v;
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(p);
+
+  if (res < min_val) {
+    log_warn(LD_DIR, "Consensus parameter %s is too small. Got %d, raising to "
+             "%d.", param_name, res, min_val);
+    res = min_val;
+  } else if (res > max_val) {
+    log_warn(LD_DIR, "Consensus parameter %s is too large. Got %d, capping to "
+             "%d.", param_name, res, max_val);
+    res = max_val;
+  }
+
+  return res;
+}
+
+/** Return the value of a integer parameter from the networkstatus <b>ns</b>
+ * whose name is <b>param_name</b>.  If <b>ns</b> is NULL, try loading the
+ * latest consensus ourselves. Return <b>default_val</b> if no latest
+ * consensus, or if it has no parameter called <b>param_name</b>.
+ * Make sure the value parsed from the consensus is at least
+ * <b>min_val</b> and at most <b>max_val</b> and raise/cap the parsed value
+ * if necessary. */
+MOCK_IMPL(int32_t,
+networkstatus_get_param, (const networkstatus_t *ns, const char *param_name,
+                        int32_t default_val, int32_t min_val, int32_t max_val))
+{
+  if (!ns) /* if they pass in null, go find it ourselves */
+    ns = networkstatus_get_latest_consensus();
+
+  if (!ns || !ns->net_params)
+    return default_val;
+
+  return get_net_param_from_list(ns->net_params, param_name,
+                                 default_val, min_val, max_val);
+}
+
+/**
+ * As networkstatus_get_param(), but check torrc_value before checking the
+ * consensus. If torrc_value is in-range, then return it instead of the
+ * value from the consensus.
+ */
+int32_t
+networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                    int32_t torrc_value,
+                                    const char *param_name,
+                                    int32_t default_val,
+                                    int32_t min_val, int32_t max_val)
+{
+  if (torrc_value >= min_val && torrc_value <= max_val)
+    return torrc_value;
+  else
+    return networkstatus_get_param(
+                         ns, param_name, default_val, min_val, max_val);
+}
+
+/**
+ * Retrieve the consensus parameter that governs the
+ * fixed-point precision of our network balancing 'bandwidth-weights'
+ * (which are themselves integer consensus values). We divide them
+ * by this value and ensure they never exceed this value.
+ */
+int
+networkstatus_get_weight_scale_param(networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "bwweightscale",
+                                 BW_WEIGHT_SCALE,
+                                 BW_MIN_WEIGHT_SCALE,
+                                 BW_MAX_WEIGHT_SCALE);
+}
+
+/** Return the value of a integer bw weight parameter from the networkstatus
+ * <b>ns</b> whose name is <b>weight_name</b>.  If <b>ns</b> is NULL, try
+ * loading the latest consensus ourselves. Return <b>default_val</b> if no
+ * latest consensus, or if it has no parameter called <b>weight_name</b>. */
+int32_t
+networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight_name,
+                            int32_t default_val)
+{
+  int32_t param;
+  int max;
+  if (!ns) /* if they pass in null, go find it ourselves */
+    ns = networkstatus_get_latest_consensus();
+
+  if (!ns || !ns->weight_params)
+    return default_val;
+
+  max = networkstatus_get_weight_scale_param(ns);
+  param = get_net_param_from_list(ns->weight_params, weight_name,
+                                  default_val, -1,
+                                  BW_MAX_WEIGHT_SCALE);
+  if (param > max) {
+    log_warn(LD_DIR, "Value of consensus weight %s was too large, capping "
+             "to %d", weight_name, max);
+    param = max;
+  }
+  return param;
+}
+
+/** Return the name of the consensus flavor <b>flav</b> as used to identify
+ * the flavor in directory documents. */
+const char *
+networkstatus_get_flavor_name(consensus_flavor_t flav)
+{
+  switch (flav) {
+    case FLAV_NS:
+      return "ns";
+    case FLAV_MICRODESC:
+      return "microdesc";
+    default:
+      tor_fragile_assert();
+      return "??";
+  }
+}
+
+/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>,
+ * or -1 if the flavor is not recognized. */
+int
+networkstatus_parse_flavor_name(const char *flavname)
+{
+  if (!strcmp(flavname, "ns"))
+    return FLAV_NS;
+  else if (!strcmp(flavname, "microdesc"))
+    return FLAV_MICRODESC;
+  else
+    return -1;
+}
+
+/** Return 0 if this routerstatus is obsolete, too new, isn't
+ * running, or otherwise not a descriptor that we would make any
+ * use of even if we had it. Else return 1. */
+int
+client_would_use_router(const routerstatus_t *rs, time_t now)
+{
+  if (!rs->is_flagged_running) {
+    /* If we had this router descriptor, we wouldn't even bother using it.
+     * (Fetching and storing depends on by we_want_to_fetch_flavor().) */
+    return 0;
+  }
+  if (rs->published_on + OLD_ROUTER_DESC_MAX_AGE < now) {
+    /* We'd drop it immediately for being too old. */
+    return 0;
+  }
+  if (!routerstatus_version_supports_extend2_cells(rs, 1)) {
+    /* We'd ignore it because it doesn't support EXTEND2 cells.
+     * If we don't know the version, download the descriptor so we can
+     * check if it supports EXTEND2 cells and ntor. */
+    return 0;
+  }
+  return 1;
+}
+
+/** If <b>question</b> is a string beginning with "ns/" in a format the
+ * control interface expects for a GETINFO question, set *<b>answer</b> to a
+ * newly-allocated string containing networkstatus lines for the appropriate
+ * ORs.  Return 0 on success, -1 on unrecognized question format. */
+int
+getinfo_helper_networkstatus(control_connection_t *conn,
+                             const char *question, char **answer,
+                             const char **errmsg)
+{
+  const routerstatus_t *status;
+  (void) conn;
+
+  if (!networkstatus_get_latest_consensus()) {
+    *answer = tor_strdup("");
+    return 0;
+  }
+
+  if (!strcmp(question, "ns/all")) {
+    smartlist_t *statuses = smartlist_new();
+    SMARTLIST_FOREACH(networkstatus_get_latest_consensus()->routerstatus_list,
+                      const routerstatus_t *, rs,
+      {
+        smartlist_add(statuses, networkstatus_getinfo_helper_single(rs));
+      });
+    *answer = smartlist_join_strings(statuses, "", 0, NULL);
+    SMARTLIST_FOREACH(statuses, char *, cp, tor_free(cp));
+    smartlist_free(statuses);
+    return 0;
+  } else if (!strcmpstart(question, "ns/id/")) {
+    char d[DIGEST_LEN];
+    const char *q = question + 6;
+    if (*q == '$')
+      ++q;
+
+    if (base16_decode(d, DIGEST_LEN, q, strlen(q)) != DIGEST_LEN) {
+      *errmsg = "Data not decodeable as hex";
+      return -1;
+    }
+    status = router_get_consensus_status_by_id(d);
+  } else if (!strcmpstart(question, "ns/name/")) {
+    const node_t *n = node_get_by_nickname(question+8, 0);
+    status = n ? n->rs : NULL;
+  } else if (!strcmpstart(question, "ns/purpose/")) {
+    *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL));
+    return *answer ? 0 : -1;
+  } else if (!strcmp(question, "consensus/packages")) {
+    const networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns && ns->package_lines)
+      *answer = smartlist_join_strings(ns->package_lines, "\n", 0, NULL);
+    else
+      *errmsg = "No consensus available";
+    return *answer ? 0 : -1;
+  } else if (!strcmp(question, "consensus/valid-after") ||
+             !strcmp(question, "consensus/fresh-until") ||
+             !strcmp(question, "consensus/valid-until")) {
+    const networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns) {
+      time_t t;
+      if (!strcmp(question, "consensus/valid-after"))
+        t = ns->valid_after;
+      else if (!strcmp(question, "consensus/fresh-until"))
+        t = ns->fresh_until;
+      else
+        t = ns->valid_until;
+
+      char tbuf[ISO_TIME_LEN+1];
+      format_iso_time(tbuf, t);
+      *answer = tor_strdup(tbuf);
+    } else {
+      *errmsg = "No consensus available";
+    }
+    return *answer ? 0 : -1;
+  } else {
+    return 0;
+  }
+
+  if (status)
+    *answer = networkstatus_getinfo_helper_single(status);
+  return 0;
+}
+
+/** Check whether the networkstatus <b>ns</b> lists any protocol
+ * versions as "required" or "recommended" that we do not support.  If
+ * so, set *<b>warning_out</b> to a newly allocated string describing
+ * the problem.
+ *
+ * Return 1 if we should exit, 0 if we should not. */
+int
+networkstatus_check_required_protocols(const networkstatus_t *ns,
+                                       int client_mode,
+                                       char **warning_out)
+{
+  const char *func = client_mode ? "client" : "relay";
+  const char *required, *recommended;
+  char *missing = NULL;
+
+  tor_assert(warning_out);
+
+  if (client_mode) {
+    required = ns->required_client_protocols;
+    recommended = ns->recommended_client_protocols;
+  } else {
+    required = ns->required_relay_protocols;
+    recommended = ns->recommended_relay_protocols;
+  }
+
+  if (!protover_all_supported(required, &missing)) {
+    tor_asprintf(warning_out, "At least one protocol listed as required in "
+                 "the consensus is not supported by this version of Tor. "
+                 "You should upgrade. This version of Tor will not work as a "
+                 "%s on the Tor network. The missing protocols are: %s",
+                 func, missing);
+    tor_free(missing);
+    return 1;
+  }
+
+  if (! protover_all_supported(recommended, &missing)) {
+    tor_asprintf(warning_out, "At least one protocol listed as recommended in "
+                 "the consensus is not supported by this version of Tor. "
+                 "You should upgrade. This version of Tor will eventually "
+                 "stop working as a %s on the Tor network. The missing "
+                 "protocols are: %s",
+                 func, missing);
+    tor_free(missing);
+  }
+
+  tor_assert_nonfatal(missing == NULL);
+
+  return 0;
+}
+
+/** Release all storage held in <b>s</b>. */
+void
+ns_detached_signatures_free_(ns_detached_signatures_t *s)
+{
+  if (!s)
+    return;
+  if (s->signatures) {
+    STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) {
+      SMARTLIST_FOREACH(sigs, document_signature_t *, sig,
+                        document_signature_free(sig));
+      smartlist_free(sigs);
+    } STRMAP_FOREACH_END;
+    strmap_free(s->signatures, NULL);
+    strmap_free(s->digests, tor_free_);
+  }
+
+  tor_free(s);
+}
+
+/** Free all storage held locally in this module. */
+void
+networkstatus_free_all(void)
+{
+  int i;
+  networkstatus_vote_free(current_ns_consensus);
+  networkstatus_vote_free(current_md_consensus);
+  current_md_consensus = current_ns_consensus = NULL;
+
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
+    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    if (waiting->consensus) {
+      networkstatus_vote_free(waiting->consensus);
+      waiting->consensus = NULL;
+    }
+    tor_free(waiting->body);
+  }
+}
diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h
new file mode 100644
index 0000000000..cc6badf0b2
--- /dev/null
+++ b/src/feature/nodelist/networkstatus.h
@@ -0,0 +1,162 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file networkstatus.h
+ * \brief Header file for networkstatus.c.
+ **/
+
+#ifndef TOR_NETWORKSTATUS_H
+#define TOR_NETWORKSTATUS_H
+
+#include "lib/testsupport/testsupport.h"
+
+void networkstatus_reset_warnings(void);
+void networkstatus_reset_download_failures(void);
+char *networkstatus_read_cached_consensus(const char *flavorname);
+int router_reload_consensus_networkstatus(void);
+void routerstatus_free_(routerstatus_t *rs);
+#define routerstatus_free(rs) \
+  FREE_AND_NULL(routerstatus_t, routerstatus_free_, (rs))
+void networkstatus_vote_free_(networkstatus_t *ns);
+#define networkstatus_vote_free(ns) \
+  FREE_AND_NULL(networkstatus_t, networkstatus_vote_free_, (ns))
+void ns_detached_signatures_free_(ns_detached_signatures_t *s);
+#define ns_detached_signatures_free(s) \
+  FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
+networkstatus_voter_info_t *networkstatus_get_voter_by_id(
+                                       networkstatus_t *vote,
+                                       const char *identity);
+document_signature_t *networkstatus_get_voter_sig_by_alg(
+                                    const networkstatus_voter_info_t *voter,
+                                    digest_algorithm_t alg);
+
+int networkstatus_check_consensus_signature(networkstatus_t *consensus,
+                                            int warn);
+int networkstatus_check_document_signature(const networkstatus_t *consensus,
+                                           document_signature_t *sig,
+                                           const authority_cert_t *cert);
+int compare_digest_to_routerstatus_entry(const void *_key,
+                                         const void **_member);
+int compare_digest_to_vote_routerstatus_entry(const void *_key,
+                                              const void **_member);
+const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns,
+                                              const char *digest);
+routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns,
+                                              const char *digest);
+int networkstatus_vote_find_entry_idx(networkstatus_t *ns,
+                                      const char *digest, int *found_out);
+
+MOCK_DECL(download_status_t *,
+  networkstatus_get_dl_status_by_flavor,
+  (consensus_flavor_t flavor));
+MOCK_DECL(download_status_t *,
+  networkstatus_get_dl_status_by_flavor_bootstrap,
+  (consensus_flavor_t flavor));
+MOCK_DECL(download_status_t *,
+  networkstatus_get_dl_status_by_flavor_running,
+  (consensus_flavor_t flavor));
+
+MOCK_DECL(smartlist_t *, router_get_descriptor_digests, (void));
+MOCK_DECL(download_status_t *,router_get_dl_status_by_descriptor_digest,
+          (const char *d));
+
+const routerstatus_t *router_get_consensus_status_by_id(const char *digest);
+routerstatus_t *router_get_mutable_consensus_status_by_id(
+                                   const char *digest);
+const routerstatus_t *router_get_consensus_status_by_descriptor_digest(
+                                   networkstatus_t *consensus,
+                                   const char *digest);
+MOCK_DECL(routerstatus_t *,
+          router_get_mutable_consensus_status_by_descriptor_digest,
+          (networkstatus_t *consensus, const char *digest));
+int we_want_to_fetch_flavor(const or_options_t *options, int flavor);
+int we_want_to_fetch_unknown_auth_certs(const or_options_t *options);
+void networkstatus_consensus_download_failed(int status_code,
+                                             const char *flavname);
+void update_consensus_networkstatus_fetch_time(time_t now);
+int should_delay_dir_fetches(const or_options_t *options,const char **msg_out);
+void update_networkstatus_downloads(time_t now);
+void update_certificate_downloads(time_t now);
+int consensus_is_waiting_for_certs(void);
+int client_would_use_router(const routerstatus_t *rs, time_t now);
+MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void));
+MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor,
+          (consensus_flavor_t f));
+MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
+int networkstatus_is_live(const networkstatus_t *ns, time_t now);
+int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                            time_t now);
+int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                                 time_t now);
+networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now,
+                                                             int flavor);
+MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now));
+int networkstatus_consensus_can_use_multiple_directories(
+                                                const or_options_t *options);
+MOCK_DECL(int, networkstatus_consensus_can_use_extra_fallbacks,(
+                                                const or_options_t *options));
+int networkstatus_consensus_is_already_downloading(const char *resource);
+int networkstatus_consensus_has_ipv6(const or_options_t* options);
+
+#define NSSET_FROM_CACHE 1
+#define NSSET_WAS_WAITING_FOR_CERTS 2
+#define NSSET_DONT_DOWNLOAD_CERTS 4
+#define NSSET_ACCEPT_OBSOLETE 8
+#define NSSET_REQUIRE_FLAVOR 16
+int networkstatus_set_current_consensus(const char *consensus,
+                                        const char *flavor,
+                                        unsigned flags,
+                                        const char *source_dir);
+void networkstatus_note_certs_arrived(const char *source_dir);
+void routers_update_all_from_networkstatus(time_t now, int dir_version);
+void routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
+                                                        int reset_failures);
+void signed_descs_update_status_from_consensus_networkstatus(
+                                                         smartlist_t *descs);
+
+char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
+char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
+void networkstatus_dump_bridge_status_to_file(time_t now);
+MOCK_DECL(int32_t, networkstatus_get_param,
+          (const networkstatus_t *ns, const char *param_name,
+           int32_t default_val, int32_t min_val, int32_t max_val));
+int32_t networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                            int32_t torrc_value,
+                                            const char *param_name,
+                                            int32_t default_val,
+                                            int32_t min_val, int32_t max_val);
+int getinfo_helper_networkstatus(control_connection_t *conn,
+                                 const char *question, char **answer,
+                                 const char **errmsg);
+int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight,
+                                    int32_t default_val);
+const char *networkstatus_get_flavor_name(consensus_flavor_t flav);
+int networkstatus_parse_flavor_name(const char *flavname);
+void document_signature_free_(document_signature_t *sig);
+#define document_signature_free(sig) \
+  FREE_AND_NULL(document_signature_t, document_signature_free_, (sig))
+document_signature_t *document_signature_dup(const document_signature_t *sig);
+void networkstatus_free_all(void);
+int networkstatus_get_weight_scale_param(networkstatus_t *ns);
+
+void vote_routerstatus_free_(vote_routerstatus_t *rs);
+#define vote_routerstatus_free(rs) \
+  FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
+
+#ifdef NETWORKSTATUS_PRIVATE
+#ifdef TOR_UNIT_TESTS
+STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
+                                                const char *flavor);
+STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                                 time_t now);
+extern networkstatus_t *current_ns_consensus;
+extern networkstatus_t *current_md_consensus;
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(NETWORKSTATUS_PRIVATE) */
+
+#endif /* !defined(TOR_NETWORKSTATUS_H) */
+
diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h
new file mode 100644
index 0000000000..6c937a75f5
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_sr_info_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_SR_INFO_ST_H
+#define NETWORKSTATUS_SR_INFO_ST_H
+
+struct networkstatus_sr_info_t {
+  /* Indicate if the dirauth partitipates in the SR protocol with its vote.
+   * This is tied to the SR flag in the vote. */
+  unsigned int participate:1;
+  /* Both vote and consensus: Current and previous SRV. If list is empty,
+   * this means none were found in either the consensus or vote. */
+  struct sr_srv_t *previous_srv;
+  struct sr_srv_t *current_srv;
+  /* Vote only: List of commitments. */
+  smartlist_t *commits;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h
new file mode 100644
index 0000000000..4a193ad149
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_st.h
@@ -0,0 +1,101 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_ST_H
+#define NETWORKSTATUS_ST_H
+
+#include "or/networkstatus_sr_info_st.h"
+
+/** Enumerates the possible seriousness values of a networkstatus document. */
+typedef enum networkstatus_type_t {
+  NS_TYPE_VOTE,
+  NS_TYPE_CONSENSUS,
+  NS_TYPE_OPINION,
+} networkstatus_type_t;
+
+/** A common structure to hold a v3 network status vote, or a v3 network
+ * status consensus. */
+struct networkstatus_t {
+  networkstatus_type_t type; /**< Vote, consensus, or opinion? */
+  consensus_flavor_t flavor; /**< If a consensus, what kind? */
+  unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains
+                                     * measured= bandwidth values. */
+
+  time_t published; /**< Vote only: Time when vote was written. */
+  time_t valid_after; /**< Time after which this vote or consensus applies. */
+  time_t fresh_until; /**< Time before which this is the most recent vote or
+                       * consensus. */
+  time_t valid_until; /**< Time after which this vote or consensus should not
+                       * be used. */
+
+  /** Consensus only: what method was used to produce this consensus? */
+  int consensus_method;
+  /** Vote only: what methods is this voter willing to use? */
+  smartlist_t *supported_methods;
+
+  /** List of 'package' lines describing hashes of downloadable packages */
+  smartlist_t *package_lines;
+
+  /** How long does this vote/consensus claim that authorities take to
+   * distribute their votes to one another? */
+  int vote_seconds;
+  /** How long does this vote/consensus claim that authorities take to
+   * distribute their consensus signatures to one another? */
+  int dist_seconds;
+
+  /** Comma-separated list of recommended client software, or NULL if this
+   * voter has no opinion. */
+  char *client_versions;
+  char *server_versions;
+
+  /** Lists of subprotocol versions which are _recommended_ for relays and
+   * clients, or which are _require_ for relays and clients. Tor shouldn't
+   * make any more network connections if a required protocol is missing.
+   */
+  char *recommended_relay_protocols;
+  char *recommended_client_protocols;
+  char *required_relay_protocols;
+  char *required_client_protocols;
+
+  /** List of flags that this vote/consensus applies to routers.  If a flag is
+   * not listed here, the voter has no opinion on what its value should be. */
+  smartlist_t *known_flags;
+
+  /** List of key=value strings for the parameters in this vote or
+   * consensus, sorted by key. */
+  smartlist_t *net_params;
+
+  /** List of key=value strings for the bw weight parameters in the
+   * consensus. */
+  smartlist_t *weight_params;
+
+  /** List of networkstatus_voter_info_t.  For a vote, only one element
+   * is included.  For a consensus, one element is included for every voter
+   * whose vote contributed to the consensus. */
+  smartlist_t *voters;
+
+  struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */
+
+  /** Digests of this document, as signed. */
+  common_digests_t digests;
+  /** A SHA3-256 digest of the document, not including signatures: used for
+   * consensus diffs */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
+
+  /** List of router statuses, sorted by identity digest.  For a vote,
+   * the elements are vote_routerstatus_t; for a consensus, the elements
+   * are routerstatus_t. */
+  smartlist_t *routerstatus_list;
+
+  /** If present, a map from descriptor digest to elements of
+   * routerstatus_list. */
+  digestmap_t *desc_digest_map;
+
+  /** Contains the shared random protocol data from a vote or consensus. */
+  networkstatus_sr_info_t sr_info;
+};
+
+#endif
diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h
new file mode 100644
index 0000000000..93ff3cd418
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_voter_info_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_VOTER_INFO_ST_H
+#define NETWORKSTATUS_VOTER_INFO_ST_H
+
+/** Information about a single voter in a vote or a consensus. */
+struct networkstatus_voter_info_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
+  char *nickname; /**< Nickname of this voter */
+  /** Digest of this voter's "legacy" identity key, if any.  In vote only; for
+   * consensuses, we treat legacy keys as additional signers. */
+  char legacy_id_digest[DIGEST_LEN];
+  char *address; /**< Address of this voter, in string format. */
+  uint32_t addr; /**< Address of this voter, in IPv4, in host order. */
+  uint16_t dir_port; /**< Directory port of this voter */
+  uint16_t or_port; /**< OR port of this voter */
+  char *contact; /**< Contact information for this voter. */
+  char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */
+
+  /* Nothing from here on is signed. */
+  /** The signature of the document and the signature's status. */
+  smartlist_t *sigs;
+};
+
+#endif
diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h
new file mode 100644
index 0000000000..d56ce27884
--- /dev/null
+++ b/src/feature/nodelist/node_st.h
@@ -0,0 +1,102 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NODE_ST_H
+#define NODE_ST_H
+
+#include "or/hsdir_index_st.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/** A node_t represents a Tor router.
+ *
+ * Specifically, a node_t is a Tor router as we are using it: a router that
+ * we are considering for circuits, connections, and so on.  A node_t is a
+ * thin wrapper around the routerstatus, routerinfo, and microdesc for a
+ * single router, and provides a consistent interface for all of them.
+ *
+ * Also, a node_t has mutable state.  While a routerinfo, a routerstatus,
+ * and a microdesc have[*] only the information read from a router
+ * descriptor, a consensus entry, and a microdescriptor (respectively)...
+ * a node_t has flags based on *our own current opinion* of the node.
+ *
+ * [*] Actually, there is some leftover information in each that is mutable.
+ *  We should try to excise that.
+ */
+struct node_t {
+  /* Indexing information */
+
+  /** Used to look up the node_t by its identity digest. */
+  HT_ENTRY(node_t) ht_ent;
+  /** Used to look up the node_t by its ed25519 identity digest. */
+  HT_ENTRY(node_t) ed_ht_ent;
+  /** Position of the node within the list of nodes */
+  int nodelist_idx;
+
+  /** The identity digest of this node_t.  No more than one node_t per
+   * identity may exist at a time. */
+  char identity[DIGEST_LEN];
+
+  /** The ed25519 identity of this node_t. This field is nonzero iff we
+   * currently have an ed25519 identity for this node in either md or ri,
+   * _and_ this node has been inserted to the ed25519-to-node map in the
+   * nodelist.
+   */
+  ed25519_public_key_t ed25519_id;
+
+  microdesc_t *md;
+  routerinfo_t *ri;
+  routerstatus_t *rs;
+
+  /* local info: copied from routerstatus, then possibly frobbed based
+   * on experience.  Authorities set this stuff directly.  Note that
+   * these reflect knowledge of the primary (IPv4) OR port only.  */
+
+  unsigned int is_running:1; /**< As far as we know, is this OR currently
+                              * running? */
+  unsigned int is_valid:1; /**< Has a trusted dirserver validated this OR?
+                            *  (For Authdir: Have we validated this OR?) */
+  unsigned int is_fast:1; /** Do we think this is a fast OR? */
+  unsigned int is_stable:1; /** Do we think this is a stable OR? */
+  unsigned int is_possible_guard:1; /**< Do we think this is an OK guard? */
+  unsigned int is_exit:1; /**< Do we think this is an OK exit? */
+  unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked,
+                               * or otherwise nasty? */
+  unsigned int is_hs_dir:1; /**< True iff this router is a hidden service
+                             * directory according to the authorities. */
+
+  /* Local info: warning state. */
+
+  unsigned int name_lookup_warned:1; /**< Have we warned the user for referring
+                                      * to this (unnamed) router by nickname?
+                                      */
+
+  /** Local info: we treat this node as if it rejects everything */
+  unsigned int rejects_all:1;
+
+  /* Local info: derived. */
+
+  /** True if the IPv6 OR port is preferred over the IPv4 OR port.
+   * XX/teor - can this become out of date if the torrc changes? */
+  unsigned int ipv6_preferred:1;
+
+  /** According to the geoip db what country is this router in? */
+  /* XXXprop186 what is this suppose to mean with multiple OR ports? */
+  country_t country;
+
+  /* The below items are used only by authdirservers for
+   * reachability testing. */
+
+  /** When was the last time we could reach this OR? */
+  time_t last_reachable;        /* IPv4. */
+  time_t last_reachable6;       /* IPv6. */
+
+  /* Hidden service directory index data. This is used by a service or client
+   * in order to know what's the hs directory index for this node at the time
+   * the consensus is set. */
+  struct hsdir_index_t hsdir_index;
+};
+
+#endif
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
new file mode 100644
index 0000000000..51fd0015df
--- /dev/null
+++ b/src/feature/nodelist/nodelist.c
@@ -0,0 +1,2513 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodelist.c
+ *
+ * \brief Structures and functions for tracking what we know about the routers
+ *   on the Tor network, and correlating information from networkstatus,
+ *   routerinfo, and microdescs.
+ *
+ * The key structure here is node_t: that's the canonical way to refer
+ * to a Tor relay that we might want to build a circuit through.  Every
+ * node_t has either a routerinfo_t, or a routerstatus_t from the current
+ * networkstatus consensus.  If it has a routerstatus_t, it will also
+ * need to have a microdesc_t before you can use it for circuits.
+ *
+ * The nodelist_t is a global singleton that maps identities to node_t
+ * objects.  Access them with the node_get_*() functions.  The nodelist_t
+ * is maintained by calls throughout the codebase
+ *
+ * Generally, other code should not have to reach inside a node_t to
+ * see what information it has.  Instead, you should call one of the
+ * many accessor functions that works on a generic node_t.  If there
+ * isn't one that does what you need, it's better to make such a function,
+ * and then use it.
+ *
+ * For historical reasons, some of the functions that select a node_t
+ * from the list of all usable node_t objects are in the routerlist.c
+ * module, since they originally selected a routerinfo_t. (TODO: They
+ * should move!)
+ *
+ * (TODO: Perhaps someday we should abstract the remaining ways of
+ * talking about a relay to also be node_t instances. Those would be
+ * routerstatus_t as used for directory requests, and dir_server_t as
+ * used for authorities and fallback directories.)
+ */
+
+#define NODELIST_PRIVATE
+
+#include "or/or.h"
+#include "lib/net/address.h"
+#include "or/address_set.h"
+#include "or/bridges.h"
+#include "or/config.h"
+#include "or/control.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "or/geoip.h"
+#include "or/hs_common.h"
+#include "or/hs_client.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/protover.h"
+#include "or/rendservice.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "or/torcert.h"
+
+#include <string.h>
+
+#include "or/dirauth/mode.h"
+
+#include "or/dir_server_st.h"
+#include "or/microdesc_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/routerstatus_st.h"
+
+static void nodelist_drop_node(node_t *node, int remove_from_ht);
+#define node_free(val) \
+  FREE_AND_NULL(node_t, node_free_, (val))
+static void node_free_(node_t *node);
+
+/** count_usable_descriptors counts descriptors with these flag(s)
+ */
+typedef enum {
+  /* All descriptors regardless of flags */
+  USABLE_DESCRIPTOR_ALL = 0,
+  /* Only descriptors with the Exit flag */
+  USABLE_DESCRIPTOR_EXIT_ONLY = 1
+} usable_descriptor_t;
+static void count_usable_descriptors(int *num_present,
+                                     int *num_usable,
+                                     smartlist_t *descs_out,
+                                     const networkstatus_t *consensus,
+                                     time_t now,
+                                     routerset_t *in_set,
+                                     usable_descriptor_t exit_only);
+static void update_router_have_minimum_dir_info(void);
+static double get_frac_paths_needed_for_circs(const or_options_t *options,
+                                              const networkstatus_t *ns);
+static void node_add_to_address_set(const node_t *node);
+
+/** A nodelist_t holds a node_t object for every router we're "willing to use
+ * for something".  Specifically, it should hold a node_t for every node that
+ * is currently in the routerlist, or currently in the consensus we're using.
+ */
+typedef struct nodelist_t {
+  /* A list of all the nodes. */
+  smartlist_t *nodes;
+  /* Hash table to map from node ID digest to node. */
+  HT_HEAD(nodelist_map, node_t) nodes_by_id;
+  /* Hash table to map from node Ed25519 ID to node.
+   *
+   * Whenever a node's routerinfo or microdescriptor is about to change,
+   * you should remove it from this map with node_remove_from_ed25519_map().
+   * Whenever a node's routerinfo or microdescriptor has just chaned,
+   * you should add it to this map with node_add_to_ed25519_map().
+   */
+  HT_HEAD(nodelist_ed_map, node_t) nodes_by_ed_id;
+
+  /* Set of addresses that belong to nodes we believe in. */
+  address_set_t *node_addrs;
+
+  /* The valid-after time of the last live consensus that initialized the
+   * nodelist.  We use this to detect outdated nodelists that need to be
+   * rebuilt using a newer consensus. */
+  time_t live_consensus_valid_after;
+} nodelist_t;
+
+static inline unsigned int
+node_id_hash(const node_t *node)
+{
+  return (unsigned) siphash24g(node->identity, DIGEST_LEN);
+}
+
+static inline unsigned int
+node_id_eq(const node_t *node1, const node_t *node2)
+{
+  return tor_memeq(node1->identity, node2->identity, DIGEST_LEN);
+}
+
+HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq)
+HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq,
+             0.6, tor_reallocarray_, tor_free_)
+
+static inline unsigned int
+node_ed_id_hash(const node_t *node)
+{
+  return (unsigned) siphash24g(node->ed25519_id.pubkey, ED25519_PUBKEY_LEN);
+}
+
+static inline unsigned int
+node_ed_id_eq(const node_t *node1, const node_t *node2)
+{
+  return ed25519_pubkey_eq(&node1->ed25519_id, &node2->ed25519_id);
+}
+
+HT_PROTOTYPE(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq)
+HT_GENERATE2(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** The global nodelist. */
+static nodelist_t *the_nodelist=NULL;
+
+/** Create an empty nodelist if we haven't done so already. */
+static void
+init_nodelist(void)
+{
+  if (PREDICT_UNLIKELY(the_nodelist == NULL)) {
+    the_nodelist = tor_malloc_zero(sizeof(nodelist_t));
+    HT_INIT(nodelist_map, &the_nodelist->nodes_by_id);
+    HT_INIT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
+    the_nodelist->nodes = smartlist_new();
+  }
+}
+
+/** As node_get_by_id, but returns a non-const pointer */
+MOCK_IMPL(node_t *,
+node_get_mutable_by_id,(const char *identity_digest))
+{
+  node_t search, *node;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return NULL;
+
+  memcpy(&search.identity, identity_digest, DIGEST_LEN);
+  node = HT_FIND(nodelist_map, &the_nodelist->nodes_by_id, &search);
+  return node;
+}
+
+/** As node_get_by_ed25519_id, but returns a non-const pointer */
+node_t *
+node_get_mutable_by_ed25519_id(const ed25519_public_key_t *ed_id)
+{
+  node_t search, *node;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return NULL;
+  if (BUG(ed_id == NULL) || BUG(ed25519_public_key_is_zero(ed_id)))
+    return NULL;
+
+  memcpy(&search.ed25519_id, ed_id, sizeof(search.ed25519_id));
+  node = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, &search);
+  return node;
+}
+
+/** Return the node_t whose identity is <b>identity_digest</b>, or NULL
+ * if no such node exists. */
+MOCK_IMPL(const node_t *,
+node_get_by_id,(const char *identity_digest))
+{
+  return node_get_mutable_by_id(identity_digest);
+}
+
+/** Return the node_t whose ed25519 identity is <b>ed_id</b>, or NULL
+ * if no such node exists. */
+MOCK_IMPL(const node_t *,
+node_get_by_ed25519_id,(const ed25519_public_key_t *ed_id))
+{
+  return node_get_mutable_by_ed25519_id(ed_id);
+}
+
+/** Internal: return the node_t whose identity_digest is
+ * <b>identity_digest</b>.  If none exists, create a new one, add it to the
+ * nodelist, and return it.
+ *
+ * Requires that the nodelist be initialized.
+ */
+static node_t *
+node_get_or_create(const char *identity_digest)
+{
+  node_t *node;
+
+  if ((node = node_get_mutable_by_id(identity_digest)))
+    return node;
+
+  node = tor_malloc_zero(sizeof(node_t));
+  memcpy(node->identity, identity_digest, DIGEST_LEN);
+  HT_INSERT(nodelist_map, &the_nodelist->nodes_by_id, node);
+
+  smartlist_add(the_nodelist->nodes, node);
+  node->nodelist_idx = smartlist_len(the_nodelist->nodes) - 1;
+
+  node->country = -1;
+
+  return node;
+}
+
+/** Remove <b>node</b> from the ed25519 map (if it present), and
+ * set its ed25519_id field to zero. */
+static int
+node_remove_from_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  int rv = 0;
+  node_t *search =
+    HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (BUG(search != node)) {
+    goto clear_and_return;
+  }
+
+  search = HT_REMOVE(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  tor_assert(search == node);
+  rv = 1;
+
+ clear_and_return:
+  memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+  return rv;
+}
+
+/** If <b>node</b> has an ed25519 id, and it is not already in the ed25519 id
+ * map, set its ed25519_id field, and add it to the ed25519 map.
+ */
+static int
+node_add_to_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (! ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  const ed25519_public_key_t *key = node_get_ed25519_id(node);
+  if (!key) {
+    return 0;
+  }
+
+  node_t *old;
+  memcpy(&node->ed25519_id, key, sizeof(node->ed25519_id));
+  old = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (BUG(old)) {
+    /* XXXX order matters here, and this may mean that authorities aren't
+     * pinning. */
+    if (old != node)
+      memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+    return 0;
+  }
+
+  HT_INSERT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  return 1;
+}
+
+/* For a given <b>node</b> for the consensus <b>ns</b>, set the hsdir index
+ * for the node, both current and next if possible. This can only fails if the
+ * node_t ed25519 identity key can't be found which would be a bug. */
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns)
+{
+  time_t now = approx_time();
+  const ed25519_public_key_t *node_identity_pk;
+  uint8_t *fetch_srv = NULL, *store_first_srv = NULL, *store_second_srv = NULL;
+  uint64_t next_time_period_num, current_time_period_num;
+  uint64_t fetch_tp, store_first_tp, store_second_tp;
+
+  tor_assert(node);
+  tor_assert(ns);
+
+  if (!networkstatus_is_live(ns, now)) {
+    static struct ratelim_t live_consensus_ratelim = RATELIM_INIT(30 * 60);
+    log_fn_ratelim(&live_consensus_ratelim, LOG_INFO, LD_GENERAL,
+                   "Not setting hsdir index with a non-live consensus.");
+    goto done;
+  }
+
+  node_identity_pk = node_get_ed25519_id(node);
+  if (node_identity_pk == NULL) {
+    log_debug(LD_GENERAL, "ed25519 identity public key not found when "
+                          "trying to build the hsdir indexes for node %s",
+              node_describe(node));
+    goto done;
+  }
+
+  /* Get the current and next time period number. */
+  current_time_period_num = hs_get_time_period_num(0);
+  next_time_period_num = hs_get_next_time_period_num(0);
+
+  /* We always use the current time period for fetching descs */
+  fetch_tp = current_time_period_num;
+
+  /* Now extract the needed SRVs and time periods for building hsdir indices */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    fetch_srv = hs_get_current_srv(fetch_tp, ns);
+
+    store_first_tp = hs_get_previous_time_period_num(0);
+    store_second_tp = current_time_period_num;
+  } else {
+    fetch_srv = hs_get_previous_srv(fetch_tp, ns);
+
+    store_first_tp = current_time_period_num;
+    store_second_tp = next_time_period_num;
+  }
+
+  /* We always use the old SRV for storing the first descriptor and the latest
+   * SRV for storing the second descriptor */
+  store_first_srv = hs_get_previous_srv(store_first_tp, ns);
+  store_second_srv = hs_get_current_srv(store_second_tp, ns);
+
+  /* Build the fetch index. */
+  hs_build_hsdir_index(node_identity_pk, fetch_srv, fetch_tp,
+                       node->hsdir_index.fetch);
+
+  /* If we are in the time segment between SRV#N and TP#N, the fetch index is
+     the same as the first store index */
+  if (!hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_first, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_first));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_first_srv, store_first_tp,
+                         node->hsdir_index.store_first);
+  }
+
+  /* If we are in the time segment between TP#N and SRV#N+1, the fetch index is
+     the same as the second store index */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_second, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_second));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_second_srv, store_second_tp,
+                         node->hsdir_index.store_second);
+  }
+
+ done:
+  tor_free(fetch_srv);
+  tor_free(store_first_srv);
+  tor_free(store_second_srv);
+  return;
+}
+
+/** Called when a node's address changes. */
+static void
+node_addrs_changed(node_t *node)
+{
+  node->last_reachable = node->last_reachable6 = 0;
+  node->country = -1;
+}
+
+/** Add all address information about <b>node</b> to the current address
+ * set (if there is one).
+ */
+static void
+node_add_to_address_set(const node_t *node)
+{
+  if (!the_nodelist || !the_nodelist->node_addrs)
+    return;
+
+  /* These various address sources can be redundant, but it's likely faster
+   * to add them all than to compare them all for equality. */
+
+  if (node->rs) {
+    if (node->rs->addr)
+      address_set_add_ipv4h(the_nodelist->node_addrs, node->rs->addr);
+    if (!tor_addr_is_null(&node->rs->ipv6_addr))
+      address_set_add(the_nodelist->node_addrs, &node->rs->ipv6_addr);
+  }
+  if (node->ri) {
+    if (node->ri->addr)
+      address_set_add_ipv4h(the_nodelist->node_addrs, node->ri->addr);
+    if (!tor_addr_is_null(&node->ri->ipv6_addr))
+      address_set_add(the_nodelist->node_addrs, &node->ri->ipv6_addr);
+  }
+  if (node->md) {
+    if (!tor_addr_is_null(&node->md->ipv6_addr))
+      address_set_add(the_nodelist->node_addrs, &node->md->ipv6_addr);
+  }
+}
+
+/** Return true if <b>addr</b> is the address of some node in the nodelist.
+ * If not, probably return false. */
+int
+nodelist_probably_contains_address(const tor_addr_t *addr)
+{
+  if (BUG(!addr))
+    return 0;
+
+  if (!the_nodelist || !the_nodelist->node_addrs)
+    return 0;
+
+  return address_set_probably_contains(the_nodelist->node_addrs, addr);
+}
+
+/** Add <b>ri</b> to an appropriate node in the nodelist.  If we replace an
+ * old routerinfo, and <b>ri_old_out</b> is not NULL, set *<b>ri_old_out</b>
+ * to the previous routerinfo.
+ */
+node_t *
+nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
+{
+  node_t *node;
+  const char *id_digest;
+  int had_router = 0;
+  tor_assert(ri);
+
+  init_nodelist();
+  id_digest = ri->cache_info.identity_digest;
+  node = node_get_or_create(id_digest);
+
+  node_remove_from_ed25519_map(node);
+
+  if (node->ri) {
+    if (!routers_have_same_or_addrs(node->ri, ri)) {
+      node_addrs_changed(node);
+    }
+    had_router = 1;
+    if (ri_old_out)
+      *ri_old_out = node->ri;
+  } else {
+    if (ri_old_out)
+      *ri_old_out = NULL;
+  }
+  node->ri = ri;
+
+  node_add_to_ed25519_map(node);
+
+  if (node->country == -1)
+    node_set_country(node);
+
+  if (authdir_mode(get_options()) && !had_router) {
+    const char *discard=NULL;
+    uint32_t status = dirserv_router_get_status(ri, &discard, LOG_INFO);
+    dirserv_set_node_flags_from_authoritative_status(node, status);
+  }
+
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (node->rs && node->rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node,
+                         networkstatus_get_latest_consensus());
+  }
+
+  node_add_to_address_set(node);
+
+  return node;
+}
+
+/** Set the appropriate node_t to use <b>md</b> as its microdescriptor.
+ *
+ * Called when a new microdesc has arrived and the usable consensus flavor
+ * is "microdesc".
+ **/
+node_t *
+nodelist_add_microdesc(microdesc_t *md)
+{
+  networkstatus_t *ns =
+    networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+  const routerstatus_t *rs;
+  node_t *node;
+  if (ns == NULL)
+    return NULL;
+  init_nodelist();
+
+  /* Microdescriptors don't carry an identity digest, so we need to figure
+   * it out by looking up the routerstatus. */
+  rs = router_get_consensus_status_by_descriptor_digest(ns, md->digest);
+  if (rs == NULL)
+    return NULL;
+  node = node_get_mutable_by_id(rs->identity_digest);
+  if (node == NULL)
+    return NULL;
+
+  node_remove_from_ed25519_map(node);
+  if (node->md)
+    node->md->held_by_nodes--;
+
+  node->md = md;
+  md->held_by_nodes++;
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node, ns);
+  }
+  node_add_to_ed25519_map(node);
+  node_add_to_address_set(node);
+
+  return node;
+}
+
+/* Default value. */
+#define ESTIMATED_ADDRESS_PER_NODE 2
+
+/* Return the estimated number of address per node_t. This is used for the
+ * size of the bloom filter in the nodelist (node_addrs). */
+MOCK_IMPL(int,
+get_estimated_address_per_node, (void))
+{
+  return ESTIMATED_ADDRESS_PER_NODE;
+}
+
+/** Tell the nodelist that the current usable consensus is <b>ns</b>.
+ * This makes the nodelist change all of the routerstatus entries for
+ * the nodes, drop nodes that no longer have enough info to get used,
+ * and grab microdescriptors into nodes as appropriate.
+ */
+void
+nodelist_set_consensus(networkstatus_t *ns)
+{
+  const or_options_t *options = get_options();
+  int authdir = authdir_mode_v3(options);
+
+  init_nodelist();
+  if (ns->flavor == FLAV_MICRODESC)
+    (void) get_microdesc_cache(); /* Make sure it exists first. */
+
+  SMARTLIST_FOREACH(the_nodelist->nodes, node_t *, node,
+                    node->rs = NULL);
+
+  /* Conservatively estimate that every node will have 2 addresses. */
+  const int estimated_addresses = smartlist_len(ns->routerstatus_list) *
+                                  get_estimated_address_per_node();
+  address_set_free(the_nodelist->node_addrs);
+  the_nodelist->node_addrs = address_set_new(estimated_addresses);
+
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    node_t *node = node_get_or_create(rs->identity_digest);
+    node->rs = rs;
+    if (ns->flavor == FLAV_MICRODESC) {
+      if (node->md == NULL ||
+          tor_memneq(node->md->digest,rs->descriptor_digest,DIGEST256_LEN)) {
+        node_remove_from_ed25519_map(node);
+        if (node->md)
+          node->md->held_by_nodes--;
+        node->md = microdesc_cache_lookup_by_digest256(NULL,
+                                                       rs->descriptor_digest);
+        if (node->md)
+          node->md->held_by_nodes++;
+        node_add_to_ed25519_map(node);
+      }
+    }
+
+    if (rs->pv.supports_v3_hsdir) {
+      node_set_hsdir_index(node, ns);
+    }
+    node_set_country(node);
+
+    /* If we're not an authdir, believe others. */
+    if (!authdir) {
+      node->is_valid = rs->is_valid;
+      node->is_running = rs->is_flagged_running;
+      node->is_fast = rs->is_fast;
+      node->is_stable = rs->is_stable;
+      node->is_possible_guard = rs->is_possible_guard;
+      node->is_exit = rs->is_exit;
+      node->is_bad_exit = rs->is_bad_exit;
+      node->is_hs_dir = rs->is_hs_dir;
+      node->ipv6_preferred = 0;
+      if (fascist_firewall_prefer_ipv6_orport(options) &&
+          (tor_addr_is_null(&rs->ipv6_addr) == 0 ||
+           (node->md && tor_addr_is_null(&node->md->ipv6_addr) == 0)))
+        node->ipv6_preferred = 1;
+    }
+
+  } SMARTLIST_FOREACH_END(rs);
+
+  nodelist_purge();
+
+  /* Now add all the nodes we have to the address set. */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    node_add_to_address_set(node);
+  } SMARTLIST_FOREACH_END(node);
+
+  if (! authdir) {
+    SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+      /* We have no routerstatus for this router. Clear flags so we can skip
+       * it, maybe.*/
+      if (!node->rs) {
+        tor_assert(node->ri); /* if it had only an md, or nothing, purge
+                               * would have removed it. */
+        if (node->ri->purpose == ROUTER_PURPOSE_GENERAL) {
+          /* Clear all flags. */
+          node->is_valid = node->is_running = node->is_hs_dir =
+            node->is_fast = node->is_stable =
+            node->is_possible_guard = node->is_exit =
+            node->is_bad_exit = node->ipv6_preferred = 0;
+        }
+      }
+    } SMARTLIST_FOREACH_END(node);
+  }
+
+  /* If the consensus is live, note down the consensus valid-after that formed
+   * the nodelist. */
+  if (networkstatus_is_live(ns, approx_time())) {
+    the_nodelist->live_consensus_valid_after = ns->valid_after;
+  }
+}
+
+/** Return 1 iff <b>node</b> has Exit flag and no BadExit flag.
+ * Otherwise, return 0.
+ */
+int
+node_is_good_exit(const node_t *node)
+{
+  return node->is_exit && ! node->is_bad_exit;
+}
+
+/** Helper: return true iff a node has a usable amount of information*/
+static inline int
+node_is_usable(const node_t *node)
+{
+  return (node->rs) || (node->ri);
+}
+
+/** Tell the nodelist that <b>md</b> is no longer a microdescriptor for the
+ * node with <b>identity_digest</b>. */
+void
+nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md)
+{
+  node_t *node = node_get_mutable_by_id(identity_digest);
+  if (node && node->md == md) {
+    node->md = NULL;
+    md->held_by_nodes--;
+    if (! node_get_ed25519_id(node)) {
+      node_remove_from_ed25519_map(node);
+    }
+  }
+}
+
+/** Tell the nodelist that <b>ri</b> is no longer in the routerlist. */
+void
+nodelist_remove_routerinfo(routerinfo_t *ri)
+{
+  node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+  if (node && node->ri == ri) {
+    node->ri = NULL;
+    if (! node_is_usable(node)) {
+      nodelist_drop_node(node, 1);
+      node_free(node);
+    }
+  }
+}
+
+/** Remove <b>node</b> from the nodelist.  (Asserts that it was there to begin
+ * with.) */
+static void
+nodelist_drop_node(node_t *node, int remove_from_ht)
+{
+  node_t *tmp;
+  int idx;
+  if (remove_from_ht) {
+    tmp = HT_REMOVE(nodelist_map, &the_nodelist->nodes_by_id, node);
+    tor_assert(tmp == node);
+  }
+  node_remove_from_ed25519_map(node);
+
+  idx = node->nodelist_idx;
+  tor_assert(idx >= 0);
+
+  tor_assert(node == smartlist_get(the_nodelist->nodes, idx));
+  smartlist_del(the_nodelist->nodes, idx);
+  if (idx < smartlist_len(the_nodelist->nodes)) {
+    tmp = smartlist_get(the_nodelist->nodes, idx);
+    tmp->nodelist_idx = idx;
+  }
+  node->nodelist_idx = -1;
+}
+
+/** Return a newly allocated smartlist of the nodes that have <b>md</b> as
+ * their microdescriptor. */
+smartlist_t *
+nodelist_find_nodes_with_microdesc(const microdesc_t *md)
+{
+  smartlist_t *result = smartlist_new();
+
+  if (the_nodelist == NULL)
+    return result;
+
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    if (node->md == md) {
+      smartlist_add(result, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  return result;
+}
+
+/** Release storage held by <b>node</b>  */
+static void
+node_free_(node_t *node)
+{
+  if (!node)
+    return;
+  if (node->md)
+    node->md->held_by_nodes--;
+  tor_assert(node->nodelist_idx == -1);
+  tor_free(node);
+}
+
+/** Remove all entries from the nodelist that don't have enough info to be
+ * usable for anything. */
+void
+nodelist_purge(void)
+{
+  node_t **iter;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return;
+
+  /* Remove the non-usable nodes. */
+  for (iter = HT_START(nodelist_map, &the_nodelist->nodes_by_id); iter; ) {
+    node_t *node = *iter;
+
+    if (node->md && !node->rs) {
+      /* An md is only useful if there is an rs. */
+      node->md->held_by_nodes--;
+      node->md = NULL;
+    }
+
+    if (node_is_usable(node)) {
+      iter = HT_NEXT(nodelist_map, &the_nodelist->nodes_by_id, iter);
+    } else {
+      iter = HT_NEXT_RMV(nodelist_map, &the_nodelist->nodes_by_id, iter);
+      nodelist_drop_node(node, 0);
+      node_free(node);
+    }
+  }
+  nodelist_assert_ok();
+}
+
+/** Release all storage held by the nodelist. */
+void
+nodelist_free_all(void)
+{
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return;
+
+  HT_CLEAR(nodelist_map, &the_nodelist->nodes_by_id);
+  HT_CLEAR(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    node->nodelist_idx = -1;
+    node_free(node);
+  } SMARTLIST_FOREACH_END(node);
+
+  smartlist_free(the_nodelist->nodes);
+
+  address_set_free(the_nodelist->node_addrs);
+  the_nodelist->node_addrs = NULL;
+
+  tor_free(the_nodelist);
+}
+
+/** Check that the nodelist is internally consistent, and consistent with
+ * the directory info it's derived from.
+ */
+void
+nodelist_assert_ok(void)
+{
+  routerlist_t *rl = router_get_routerlist();
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  digestmap_t *dm;
+
+  if (!the_nodelist)
+    return;
+
+  dm = digestmap_new();
+
+  /* every routerinfo in rl->routers should be in the nodelist. */
+  if (rl) {
+    SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+      const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
+      tor_assert(node && node->ri == ri);
+      tor_assert(fast_memeq(ri->cache_info.identity_digest,
+                             node->identity, DIGEST_LEN));
+      tor_assert(! digestmap_get(dm, node->identity));
+      digestmap_set(dm, node->identity, (void*)node);
+    } SMARTLIST_FOREACH_END(ri);
+  }
+
+  /* every routerstatus in ns should be in the nodelist */
+  if (ns) {
+    SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+      const node_t *node = node_get_by_id(rs->identity_digest);
+      tor_assert(node && node->rs == rs);
+      tor_assert(fast_memeq(rs->identity_digest, node->identity, DIGEST_LEN));
+      digestmap_set(dm, node->identity, (void*)node);
+      if (ns->flavor == FLAV_MICRODESC) {
+        /* If it's a microdesc consensus, every entry that has a
+         * microdescriptor should be in the nodelist.
+         */
+        microdesc_t *md =
+          microdesc_cache_lookup_by_digest256(NULL, rs->descriptor_digest);
+        tor_assert(md == node->md);
+        if (md)
+          tor_assert(md->held_by_nodes >= 1);
+      }
+    } SMARTLIST_FOREACH_END(rs);
+  }
+
+  /* The nodelist should have no other entries, and its entries should be
+   * well-formed. */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    tor_assert(digestmap_get(dm, node->identity) != NULL);
+    tor_assert(node_sl_idx == node->nodelist_idx);
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Every node listed with an ed25519 identity should be listed by that
+   * identity.
+   */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    if (!ed25519_public_key_is_zero(&node->ed25519_id)) {
+      tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  node_t **idx;
+  HT_FOREACH(idx, nodelist_ed_map, &the_nodelist->nodes_by_ed_id) {
+    node_t *node = *idx;
+    tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+  }
+
+  tor_assert((long)smartlist_len(the_nodelist->nodes) ==
+             (long)HT_SIZE(&the_nodelist->nodes_by_id));
+
+  tor_assert((long)smartlist_len(the_nodelist->nodes) >=
+             (long)HT_SIZE(&the_nodelist->nodes_by_ed_id));
+
+  digestmap_free(dm, NULL);
+}
+
+/** Ensure that the nodelist has been created with the most recent consensus.
+ *  If that's not the case, make it so.  */
+void
+nodelist_ensure_freshness(networkstatus_t *ns)
+{
+  tor_assert(ns);
+
+  /* We don't even have a nodelist: this is a NOP. */
+  if (!the_nodelist) {
+    return;
+  }
+
+  if (the_nodelist->live_consensus_valid_after != ns->valid_after) {
+    log_info(LD_GENERAL, "Nodelist was not fresh: rebuilding. (%d / %d)",
+             (int) the_nodelist->live_consensus_valid_after,
+             (int) ns->valid_after);
+    nodelist_set_consensus(ns);
+  }
+}
+/** Return a list of a node_t * for every node we know about.  The caller
+ * MUST NOT modify the list. (You can set and clear flags in the nodes if
+ * you must, but you must not add or remove nodes.) */
+MOCK_IMPL(smartlist_t *,
+nodelist_get_list,(void))
+{
+  init_nodelist();
+  return the_nodelist->nodes;
+}
+
+/** Given a hex-encoded nickname of the format DIGEST, $DIGEST, $DIGEST=name,
+ * or $DIGEST~name, return the node with the matching identity digest and
+ * nickname (if any).  Return NULL if no such node exists, or if <b>hex_id</b>
+ * is not well-formed. DOCDOC flags */
+const node_t *
+node_get_by_hex_id(const char *hex_id, unsigned flags)
+{
+  char digest_buf[DIGEST_LEN];
+  char nn_buf[MAX_NICKNAME_LEN+1];
+  char nn_char='\0';
+
+  (void) flags; // XXXX
+
+  if (hex_digest_nickname_decode(hex_id, digest_buf, &nn_char, nn_buf)==0) {
+    const node_t *node = node_get_by_id(digest_buf);
+    if (!node)
+      return NULL;
+    if (nn_char == '=') {
+      /* "=" indicates a Named relay, but there aren't any of those now. */
+      return NULL;
+    }
+    return node;
+  }
+
+  return NULL;
+}
+
+/** Given a nickname (possibly verbose, possibly a hexadecimal digest), return
+ * the corresponding node_t, or NULL if none exists. Warn the user if they
+ * have specified a router by nickname, unless the NNF_NO_WARN_UNNAMED bit is
+ * set in <b>flags</b>. */
+MOCK_IMPL(const node_t *,
+node_get_by_nickname,(const char *nickname, unsigned flags))
+{
+  const int warn_if_unnamed = !(flags & NNF_NO_WARN_UNNAMED);
+
+  if (!the_nodelist)
+    return NULL;
+
+  /* Handle these cases: DIGEST, $DIGEST, $DIGEST=name, $DIGEST~name. */
+  {
+    const node_t *node;
+    if ((node = node_get_by_hex_id(nickname, flags)) != NULL)
+      return node;
+  }
+
+  if (!strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME))
+    return NULL;
+
+  /* Okay, so the name is not canonical for anybody. */
+  {
+    smartlist_t *matches = smartlist_new();
+    const node_t *choice = NULL;
+
+    SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+      if (!strcasecmp(node_get_nickname(node), nickname))
+        smartlist_add(matches, node);
+    } SMARTLIST_FOREACH_END(node);
+
+    if (smartlist_len(matches)>1 && warn_if_unnamed) {
+      int any_unwarned = 0;
+      SMARTLIST_FOREACH_BEGIN(matches, node_t *, node) {
+        if (!node->name_lookup_warned) {
+          node->name_lookup_warned = 1;
+          any_unwarned = 1;
+        }
+      } SMARTLIST_FOREACH_END(node);
+
+      if (any_unwarned) {
+        log_warn(LD_CONFIG, "There are multiple matches for the name %s, "
+                 "but none is listed as Named in the directory consensus. "
+                 "Choosing one arbitrarily.", nickname);
+      }
+    } else if (smartlist_len(matches)==1 && warn_if_unnamed) {
+      char fp[HEX_DIGEST_LEN+1];
+      node_t *node = smartlist_get(matches, 0);
+      if (! node->name_lookup_warned) {
+        base16_encode(fp, sizeof(fp), node->identity, DIGEST_LEN);
+        log_warn(LD_CONFIG,
+                 "You specified a relay \"%s\" by name, but nicknames can be "
+                 "used by any relay, not just the one you meant. "
+                 "To make sure you get the same relay in the future, refer "
+                 "to it by key, as \"$%s\".", nickname, fp);
+        node->name_lookup_warned = 1;
+      }
+    }
+
+    if (smartlist_len(matches))
+      choice = smartlist_get(matches, 0);
+
+    smartlist_free(matches);
+    return choice;
+  }
+}
+
+/** Return the Ed25519 identity key for the provided node, or NULL if it
+ * doesn't have one. */
+const ed25519_public_key_t *
+node_get_ed25519_id(const node_t *node)
+{
+  const ed25519_public_key_t *ri_pk = NULL;
+  const ed25519_public_key_t *md_pk = NULL;
+
+  if (node->ri) {
+    if (node->ri->cache_info.signing_key_cert) {
+      ri_pk = &node->ri->cache_info.signing_key_cert->signing_key;
+      /* Checking whether routerinfo ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(ri_pk)))
+        ri_pk = NULL;
+    }
+  }
+
+  if (node->md) {
+    if (node->md->ed25519_identity_pkey) {
+      md_pk = node->md->ed25519_identity_pkey;
+      /* Checking whether microdesc ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(md_pk)))
+        md_pk = NULL;
+    }
+  }
+
+  if (ri_pk && md_pk) {
+    if (ed25519_pubkey_eq(ri_pk, md_pk)) {
+      return ri_pk;
+    } else {
+      /* This can happen if the relay gets flagged NoEdConsensus which will be
+       * triggered on all relays of the network. Thus a protocol warning. */
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Inconsistent ed25519 identities in the nodelist");
+      return NULL;
+    }
+  } else if (ri_pk) {
+    return ri_pk;
+  } else {
+    return md_pk;
+  }
+}
+
+/** Return true iff this node's Ed25519 identity matches <b>id</b>.
+ * (An absent Ed25519 identity matches NULL or zero.) */
+int
+node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
+{
+  const ed25519_public_key_t *node_id = node_get_ed25519_id(node);
+  if (node_id == NULL || ed25519_public_key_is_zero(node_id)) {
+    return id == NULL || ed25519_public_key_is_zero(id);
+  } else {
+    return id && ed25519_pubkey_eq(node_id, id);
+  }
+}
+
+/** Dummy object that should be unreturnable.  Used to ensure that
+ * node_get_protover_summary_flags() always returns non-NULL. */
+static const protover_summary_flags_t zero_protover_flags = {
+  0,0,0,0,0,0,0
+};
+
+/** Return the protover_summary_flags for a given node. */
+static const protover_summary_flags_t *
+node_get_protover_summary_flags(const node_t *node)
+{
+  if (node->rs) {
+    return &node->rs->pv;
+  } else if (node->ri) {
+    return &node->ri->pv;
+  } else {
+    /* This should be impossible: every node should have a routerstatus or a
+     * router descriptor or both. But just in case we've messed up somehow,
+     * return a nice empty set of flags to indicate "this node supports
+     * nothing." */
+    tor_assert_nonfatal_unreached_once();
+    return &zero_protover_flags;
+  }
+}
+
+/** Return true iff <b>node</b> supports authenticating itself
+ * by ed25519 ID during the link handshake.  If <b>compatible_with_us</b>,
+ * it needs to be using a link authentication method that we understand.
+ * If not, any plausible link authentication method will do. */
+int
+node_supports_ed25519_link_authentication(const node_t *node,
+                                          int compatible_with_us)
+{
+  if (! node_get_ed25519_id(node))
+    return 0;
+
+  const protover_summary_flags_t *pv = node_get_protover_summary_flags(node);
+
+  if (compatible_with_us)
+    return pv->supports_ed25519_link_handshake_compat;
+  else
+    return pv->supports_ed25519_link_handshake_any;
+}
+
+/** Return true iff <b>node</b> supports the hidden service directory version
+ * 3 protocol (proposal 224). */
+int
+node_supports_v3_hsdir(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_v3_hsdir;
+}
+
+/** Return true iff <b>node</b> supports ed25519 authentication as an hidden
+ * service introduction point.*/
+int
+node_supports_ed25519_hs_intro(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
+}
+
+/** Return true iff <b>node</b> supports to be a rendezvous point for hidden
+ * service version 3 (HSRend=2). */
+int
+node_supports_v3_rendezvous_point(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_v3_rendezvous_point;
+}
+
+/** Return the RSA ID key's SHA1 digest for the provided node. */
+const uint8_t *
+node_get_rsa_id_digest(const node_t *node)
+{
+  tor_assert(node);
+  return (const uint8_t*)node->identity;
+}
+
+/** Return the nickname of <b>node</b>, or NULL if we can't find one. */
+const char *
+node_get_nickname(const node_t *node)
+{
+  tor_assert(node);
+  if (node->rs)
+    return node->rs->nickname;
+  else if (node->ri)
+    return node->ri->nickname;
+  else
+    return NULL;
+}
+
+/** Return true iff <b>node</b> appears to be a directory authority or
+ * directory cache */
+int
+node_is_dir(const node_t *node)
+{
+  if (node->rs) {
+    routerstatus_t * rs = node->rs;
+    /* This is true if supports_tunnelled_dir_requests is true which
+     * indicates that we support directory request tunnelled or through the
+     * DirPort. */
+    return rs->is_v2_dir;
+  } else if (node->ri) {
+    routerinfo_t * ri = node->ri;
+    /* Both tunnelled request is supported or DirPort is set. */
+    return ri->supports_tunnelled_dir_requests;
+  } else {
+    return 0;
+  }
+}
+
+/** Return true iff <b>node</b> has either kind of descriptor -- that
+ * is, a routerdescriptor or a microdescriptor.
+ *
+ * You should probably use node_has_preferred_descriptor() instead.
+ **/
+int
+node_has_any_descriptor(const node_t *node)
+{
+  return (node->ri ||
+          (node->rs && node->md));
+}
+
+/** Return true iff <b>node</b> has the kind of descriptor we would prefer to
+ * use for it, given our configuration and how we intend to use the node.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect to
+ * it indirectly, or use it as if we were connecting to it indirectly. */
+int
+node_has_preferred_descriptor(const node_t *node,
+                              int for_direct_connect)
+{
+  const int is_bridge = node_is_a_configured_bridge(node);
+  const int we_use_mds = we_use_microdescriptors_for_circuits(get_options());
+
+  if ((is_bridge && for_direct_connect) || !we_use_mds) {
+    /* We need an ri in this case. */
+    if (!node->ri)
+      return 0;
+  } else {
+    /* Otherwise we need an rs and an md. */
+    if (node->rs == NULL || node->md == NULL)
+      return 0;
+  }
+
+  return 1;
+}
+
+/** Return the router_purpose of <b>node</b>. */
+int
+node_get_purpose(const node_t *node)
+{
+  if (node->ri)
+    return node->ri->purpose;
+  else
+    return ROUTER_PURPOSE_GENERAL;
+}
+
+/** Compute the verbose ("extended") nickname of <b>node</b> and store it
+ * into the MAX_VERBOSE_NICKNAME_LEN+1 character buffer at
+ * <b>verbose_name_out</b> */
+void
+node_get_verbose_nickname(const node_t *node,
+                          char *verbose_name_out)
+{
+  const char *nickname = node_get_nickname(node);
+  verbose_name_out[0] = '$';
+  base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, node->identity,
+                DIGEST_LEN);
+  if (!nickname)
+    return;
+  verbose_name_out[1+HEX_DIGEST_LEN] = '~';
+  strlcpy(verbose_name_out+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
+}
+
+/** Compute the verbose ("extended") nickname of node with
+ * given <b>id_digest</b> and store it into the MAX_VERBOSE_NICKNAME_LEN+1
+ * character buffer at <b>verbose_name_out</b>
+ *
+ * If node_get_by_id() returns NULL, base 16 encoding of
+ * <b>id_digest</b> is returned instead. */
+void
+node_get_verbose_nickname_by_id(const char *id_digest,
+                                char *verbose_name_out)
+{
+  const node_t *node = node_get_by_id(id_digest);
+  if (!node) {
+    verbose_name_out[0] = '$';
+    base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
+  } else {
+    node_get_verbose_nickname(node, verbose_name_out);
+  }
+}
+
+/** Return true iff it seems that <b>node</b> allows circuits to exit
+ * through it directlry from the client. */
+int
+node_allows_single_hop_exits(const node_t *node)
+{
+  if (node && node->ri)
+    return node->ri->allow_single_hop_exits;
+  else
+    return 0;
+}
+
+/** Return true iff it seems that <b>node</b> has an exit policy that doesn't
+ * actually permit anything to exit, or we don't know its exit policy */
+int
+node_exit_policy_rejects_all(const node_t *node)
+{
+  if (node->rejects_all)
+    return 1;
+
+  if (node->ri)
+    return node->ri->policy_is_reject_star;
+  else if (node->md)
+    return node->md->exit_policy == NULL ||
+      short_policy_is_reject_star(node->md->exit_policy);
+  else
+    return 1;
+}
+
+/** Return true iff the exit policy for <b>node</b> is such that we can treat
+ * rejecting an address of type <b>family</b> unexpectedly as a sign of that
+ * node's failure. */
+int
+node_exit_policy_is_exact(const node_t *node, sa_family_t family)
+{
+  if (family == AF_UNSPEC) {
+    return 1; /* Rejecting an address but not telling us what address
+               * is a bad sign. */
+  } else if (family == AF_INET) {
+    return node->ri != NULL;
+  } else if (family == AF_INET6) {
+    return 0;
+  }
+  tor_fragile_assert();
+  return 1;
+}
+
+/* Check if the "addr" and port_field fields from r are a valid non-listening
+ * address/port. If so, set valid to true and add a newly allocated
+ * tor_addr_port_t containing "addr" and port_field to sl.
+ * "addr" is an IPv4 host-order address and port_field is a uint16_t.
+ * r is typically a routerinfo_t or routerstatus_t.
+ */
+#define SL_ADD_NEW_IPV4_AP(r, port_field, sl, valid) \
+  STMT_BEGIN \
+    if (tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \
+      valid = 1; \
+      tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); \
+      tor_addr_from_ipv4h(&ap->addr, (r)->addr); \
+      ap->port = (r)->port_field; \
+      smartlist_add((sl), ap); \
+    } \
+  STMT_END
+
+/* Check if the "addr" and port_field fields from r are a valid non-listening
+ * address/port. If so, set valid to true and add a newly allocated
+ * tor_addr_port_t containing "addr" and port_field to sl.
+ * "addr" is a tor_addr_t and port_field is a uint16_t.
+ * r is typically a routerinfo_t or routerstatus_t.
+ */
+#define SL_ADD_NEW_IPV6_AP(r, port_field, sl, valid) \
+  STMT_BEGIN \
+    if (tor_addr_port_is_valid(&(r)->ipv6_addr, (r)->port_field, 0)) { \
+      valid = 1; \
+      tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); \
+      tor_addr_copy(&ap->addr, &(r)->ipv6_addr); \
+      ap->port = (r)->port_field; \
+      smartlist_add((sl), ap); \
+    } \
+  STMT_END
+
+/** Return list of tor_addr_port_t with all OR ports (in the sense IP
+ * addr + TCP port) for <b>node</b>.  Caller must free all elements
+ * using tor_free() and free the list using smartlist_free().
+ *
+ * XXX this is potentially a memory fragmentation hog -- if on
+ * critical path consider the option of having the caller allocate the
+ * memory
+ */
+smartlist_t *
+node_get_all_orports(const node_t *node)
+{
+  smartlist_t *sl = smartlist_new();
+  int valid = 0;
+
+  /* Find a valid IPv4 address and port */
+  if (node->ri != NULL) {
+    SL_ADD_NEW_IPV4_AP(node->ri, or_port, sl, valid);
+  }
+
+  /* If we didn't find a valid address/port in the ri, try the rs */
+  if (!valid && node->rs != NULL) {
+    SL_ADD_NEW_IPV4_AP(node->rs, or_port, sl, valid);
+  }
+
+  /* Find a valid IPv6 address and port */
+  valid = 0;
+  if (node->ri != NULL) {
+    SL_ADD_NEW_IPV6_AP(node->ri, ipv6_orport, sl, valid);
+  }
+
+  if (!valid && node->rs != NULL) {
+    SL_ADD_NEW_IPV6_AP(node->rs, ipv6_orport, sl, valid);
+  }
+
+  if (!valid && node->md != NULL) {
+    SL_ADD_NEW_IPV6_AP(node->md, ipv6_orport, sl, valid);
+  }
+
+  return sl;
+}
+
+#undef SL_ADD_NEW_IPV4_AP
+#undef SL_ADD_NEW_IPV6_AP
+
+/** Wrapper around node_get_prim_orport for backward
+    compatibility.  */
+void
+node_get_addr(const node_t *node, tor_addr_t *addr_out)
+{
+  tor_addr_port_t ap;
+  node_get_prim_orport(node, &ap);
+  tor_addr_copy(addr_out, &ap.addr);
+}
+
+/** Return the host-order IPv4 address for <b>node</b>, or 0 if it doesn't
+ * seem to have one.  */
+uint32_t
+node_get_prim_addr_ipv4h(const node_t *node)
+{
+  /* Don't check the ORPort or DirPort, as this function isn't port-specific,
+   * and the node might have a valid IPv4 address, yet have a zero
+   * ORPort or DirPort.
+   */
+  if (node->ri && tor_addr_is_valid_ipv4h(node->ri->addr, 0)) {
+    return node->ri->addr;
+  } else if (node->rs && tor_addr_is_valid_ipv4h(node->rs->addr, 0)) {
+    return node->rs->addr;
+  }
+  return 0;
+}
+
+/** Copy a string representation of an IP address for <b>node</b> into
+ * the <b>len</b>-byte buffer at <b>buf</b>.  */
+void
+node_get_address_string(const node_t *node, char *buf, size_t len)
+{
+  uint32_t ipv4_addr = node_get_prim_addr_ipv4h(node);
+
+  if (tor_addr_is_valid_ipv4h(ipv4_addr, 0)) {
+    tor_addr_t addr;
+    tor_addr_from_ipv4h(&addr, ipv4_addr);
+    tor_addr_to_str(buf, &addr, len, 0);
+  } else if (len > 0) {
+    buf[0] = '\0';
+  }
+}
+
+/** Return <b>node</b>'s declared uptime, or -1 if it doesn't seem to have
+ * one. */
+long
+node_get_declared_uptime(const node_t *node)
+{
+  if (node->ri)
+    return node->ri->uptime;
+  else
+    return -1;
+}
+
+/** Return <b>node</b>'s platform string, or NULL if we don't know it. */
+const char *
+node_get_platform(const node_t *node)
+{
+  /* If we wanted, we could record the version in the routerstatus_t, since
+   * the consensus lists it.  We don't, though, so this function just won't
+   * work with microdescriptors. */
+  if (node->ri)
+    return node->ri->platform;
+  else
+    return NULL;
+}
+
+/** Return true iff <b>node</b> is one representing this router. */
+int
+node_is_me(const node_t *node)
+{
+  return router_digest_is_me(node->identity);
+}
+
+/** Return <b>node</b> declared family (as a list of names), or NULL if
+ * the node didn't declare a family. */
+const smartlist_t *
+node_get_declared_family(const node_t *node)
+{
+  if (node->ri && node->ri->declared_family)
+    return node->ri->declared_family;
+  else if (node->md && node->md->family)
+    return node->md->family;
+  else
+    return NULL;
+}
+
+/* Does this node have a valid IPv6 address?
+ * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for
+ * checking specific ports. */
+int
+node_has_ipv6_addr(const node_t *node)
+{
+  /* Don't check the ORPort or DirPort, as this function isn't port-specific,
+   * and the node might have a valid IPv6 address, yet have a zero
+   * ORPort or DirPort.
+   */
+  if (node->ri && tor_addr_is_valid(&node->ri->ipv6_addr, 0))
+    return 1;
+  if (node->rs && tor_addr_is_valid(&node->rs->ipv6_addr, 0))
+    return 1;
+  if (node->md && tor_addr_is_valid(&node->md->ipv6_addr, 0))
+    return 1;
+
+  return 0;
+}
+
+/* Does this node have a valid IPv6 ORPort? */
+int
+node_has_ipv6_orport(const node_t *node)
+{
+  tor_addr_port_t ipv6_orport;
+  node_get_pref_ipv6_orport(node, &ipv6_orport);
+  return tor_addr_port_is_valid_ap(&ipv6_orport, 0);
+}
+
+/* Does this node have a valid IPv6 DirPort? */
+int
+node_has_ipv6_dirport(const node_t *node)
+{
+  tor_addr_port_t ipv6_dirport;
+  node_get_pref_ipv6_dirport(node, &ipv6_dirport);
+  return tor_addr_port_is_valid_ap(&ipv6_dirport, 0);
+}
+
+/** Return 1 if we prefer the IPv6 address and OR TCP port of
+ * <b>node</b>, else 0.
+ *
+ *  We prefer the IPv6 address if the router has an IPv6 address,
+ *  and we can use IPv6 addresses, and:
+ *  i) the node_t says that it prefers IPv6
+ *  or
+ *  ii) the router has no IPv4 OR address.
+ *
+ * If you don't have a node, consider looking it up.
+ * If there is no node, use fascist_firewall_prefer_ipv6_orport().
+ */
+int
+node_ipv6_or_preferred(const node_t *node)
+{
+  const or_options_t *options = get_options();
+  tor_addr_port_t ipv4_addr;
+  node_assert_ok(node);
+
+  /* XX/teor - node->ipv6_preferred is set from
+   * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded.
+   */
+  node_get_prim_orport(node, &ipv4_addr);
+  if (!fascist_firewall_use_ipv6(options)) {
+    return 0;
+  } else if (node->ipv6_preferred ||
+             !tor_addr_port_is_valid_ap(&ipv4_addr, 0)) {
+    return node_has_ipv6_orport(node);
+  }
+  return 0;
+}
+
+#define RETURN_IPV4_AP(r, port_field, ap_out) \
+  STMT_BEGIN \
+    if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \
+      tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \
+      (ap_out)->port = (r)->port_field; \
+    } \
+  STMT_END
+
+/** Copy the primary (IPv4) OR port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
+node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+
+  /* Clear the address, as a safety precaution if calling functions ignore the
+   * return value */
+  tor_addr_make_null(&ap_out->addr, AF_INET);
+  ap_out->port = 0;
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address. */
+
+  RETURN_IPV4_AP(node->ri, or_port, ap_out);
+  RETURN_IPV4_AP(node->rs, or_port, ap_out);
+  /* Microdescriptors only have an IPv6 address */
+}
+
+/** Copy the preferred OR port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>.  */
+void
+node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  tor_assert(ap_out);
+
+  if (node_ipv6_or_preferred(node)) {
+    node_get_pref_ipv6_orport(node, ap_out);
+  } else {
+    /* the primary ORPort is always on IPv4 */
+    node_get_prim_orport(node, ap_out);
+  }
+}
+
+/** Copy the preferred IPv6 OR port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>. */
+void
+node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+  memset(ap_out, 0, sizeof(*ap_out));
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
+
+  if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
+                                         node->ri->ipv6_orport, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->ri->ipv6_addr);
+    ap_out->port = node->ri->ipv6_orport;
+  } else if (node->rs && tor_addr_port_is_valid(&node->rs->ipv6_addr,
+                                                 node->rs->ipv6_orport, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->rs->ipv6_addr);
+    ap_out->port = node->rs->ipv6_orport;
+  } else if (node->md && tor_addr_port_is_valid(&node->md->ipv6_addr,
+                                                 node->md->ipv6_orport, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->md->ipv6_addr);
+    ap_out->port = node->md->ipv6_orport;
+  } else {
+    tor_addr_make_null(&ap_out->addr, AF_INET6);
+    ap_out->port = 0;
+  }
+}
+
+/** Return 1 if we prefer the IPv6 address and Dir TCP port of
+ * <b>node</b>, else 0.
+ *
+ *  We prefer the IPv6 address if the router has an IPv6 address,
+ *  and we can use IPv6 addresses, and:
+ *  i) the router has no IPv4 Dir address.
+ *  or
+ *  ii) our preference is for IPv6 Dir addresses.
+ *
+ * If there is no node, use fascist_firewall_prefer_ipv6_dirport().
+ */
+int
+node_ipv6_dir_preferred(const node_t *node)
+{
+  const or_options_t *options = get_options();
+  tor_addr_port_t ipv4_addr;
+  node_assert_ok(node);
+
+  /* node->ipv6_preferred is set from fascist_firewall_prefer_ipv6_orport(),
+   * so we can't use it to determine DirPort IPv6 preference.
+   * This means that bridge clients will use IPv4 DirPorts by default.
+   */
+  node_get_prim_dirport(node, &ipv4_addr);
+  if (!fascist_firewall_use_ipv6(options)) {
+    return 0;
+  } else if (!tor_addr_port_is_valid_ap(&ipv4_addr, 0)
+      || fascist_firewall_prefer_ipv6_dirport(get_options())) {
+    return node_has_ipv6_dirport(node);
+  }
+  return 0;
+}
+
+/** Copy the primary (IPv4) Dir port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
+node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+
+  /* Clear the address, as a safety precaution if calling functions ignore the
+   * return value */
+  tor_addr_make_null(&ap_out->addr, AF_INET);
+  ap_out->port = 0;
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address. */
+
+  RETURN_IPV4_AP(node->ri, dir_port, ap_out);
+  RETURN_IPV4_AP(node->rs, dir_port, ap_out);
+  /* Microdescriptors only have an IPv6 address */
+}
+
+#undef RETURN_IPV4_AP
+
+/** Copy the preferred Dir port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>.  */
+void
+node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  tor_assert(ap_out);
+
+  if (node_ipv6_dir_preferred(node)) {
+    node_get_pref_ipv6_dirport(node, ap_out);
+  } else {
+    /* the primary DirPort is always on IPv4 */
+    node_get_prim_dirport(node, ap_out);
+  }
+}
+
+/** Copy the preferred IPv6 Dir port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>. */
+void
+node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
+
+  /* Assume IPv4 and IPv6 dirports are the same */
+  if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
+                                         node->ri->dir_port, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->ri->ipv6_addr);
+    ap_out->port = node->ri->dir_port;
+  } else if (node->rs && tor_addr_port_is_valid(&node->rs->ipv6_addr,
+                                                node->rs->dir_port, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->rs->ipv6_addr);
+    ap_out->port = node->rs->dir_port;
+  } else {
+    tor_addr_make_null(&ap_out->addr, AF_INET6);
+    ap_out->port = 0;
+  }
+}
+
+/** Return true iff <b>md</b> has a curve25519 onion key.
+ * Use node_has_curve25519_onion_key() instead of calling this directly. */
+static int
+microdesc_has_curve25519_onion_key(const microdesc_t *md)
+{
+  if (!md) {
+    return 0;
+  }
+
+  if (!md->onion_curve25519_pkey) {
+    return 0;
+  }
+
+  if (tor_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
+                      CURVE25519_PUBKEY_LEN)) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/** Return true iff <b>node</b> has a curve25519 onion key. */
+int
+node_has_curve25519_onion_key(const node_t *node)
+{
+  return node_get_curve25519_onion_key(node) != NULL;
+}
+
+/** Return the curve25519 key of <b>node</b>, or NULL if none. */
+const curve25519_public_key_t *
+node_get_curve25519_onion_key(const node_t *node)
+{
+  if (!node)
+    return NULL;
+  if (routerinfo_has_curve25519_onion_key(node->ri))
+    return node->ri->onion_curve25519_pkey;
+  else if (microdesc_has_curve25519_onion_key(node->md))
+    return node->md->onion_curve25519_pkey;
+  else
+    return NULL;
+}
+
+/** Refresh the country code of <b>ri</b>.  This function MUST be called on
+ * each router when the GeoIP database is reloaded, and on all new routers. */
+void
+node_set_country(node_t *node)
+{
+  tor_addr_t addr = TOR_ADDR_NULL;
+
+  /* XXXXipv6 */
+  if (node->rs)
+    tor_addr_from_ipv4h(&addr, node->rs->addr);
+  else if (node->ri)
+    tor_addr_from_ipv4h(&addr, node->ri->addr);
+
+  node->country = geoip_get_country_by_addr(&addr);
+}
+
+/** Set the country code of all routers in the routerlist. */
+void
+nodelist_refresh_countries(void)
+{
+  smartlist_t *nodes = nodelist_get_list();
+  SMARTLIST_FOREACH(nodes, node_t *, node,
+                    node_set_country(node));
+}
+
+/** Return true iff router1 and router2 have similar enough network addresses
+ * that we should treat them as being in the same family */
+int
+addrs_in_same_network_family(const tor_addr_t *a1,
+                             const tor_addr_t *a2)
+{
+  return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
+}
+
+/** Return true if <b>node</b>'s nickname matches <b>nickname</b>
+ * (case-insensitive), or if <b>node's</b> identity key digest
+ * matches a hexadecimal value stored in <b>nickname</b>.  Return
+ * false otherwise. */
+static int
+node_nickname_matches(const node_t *node, const char *nickname)
+{
+  const char *n = node_get_nickname(node);
+  if (n && nickname[0]!='$' && !strcasecmp(n, nickname))
+    return 1;
+  return hex_digest_nickname_matches(nickname,
+                                     node->identity,
+                                     n);
+}
+
+/** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
+static inline int
+node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
+{
+  if (!lst) return 0;
+  SMARTLIST_FOREACH(lst, const char *, name, {
+    if (node_nickname_matches(node, name))
+      return 1;
+  });
+  return 0;
+}
+
+/** Return true iff r1 and r2 are in the same family, but not the same
+ * router. */
+int
+nodes_in_same_family(const node_t *node1, const node_t *node2)
+{
+  const or_options_t *options = get_options();
+
+  /* Are they in the same family because of their addresses? */
+  if (options->EnforceDistinctSubnets) {
+    tor_addr_t a1, a2;
+    node_get_addr(node1, &a1);
+    node_get_addr(node2, &a2);
+    if (addrs_in_same_network_family(&a1, &a2))
+      return 1;
+  }
+
+  /* Are they in the same family because the agree they are? */
+  {
+    const smartlist_t *f1, *f2;
+    f1 = node_get_declared_family(node1);
+    f2 = node_get_declared_family(node2);
+    if (f1 && f2 &&
+        node_in_nickname_smartlist(f1, node2) &&
+        node_in_nickname_smartlist(f2, node1))
+      return 1;
+  }
+
+  /* Are they in the same option because the user says they are? */
+  if (options->NodeFamilySets) {
+    SMARTLIST_FOREACH(options->NodeFamilySets, const routerset_t *, rs, {
+        if (routerset_contains_node(rs, node1) &&
+            routerset_contains_node(rs, node2))
+          return 1;
+      });
+  }
+
+  return 0;
+}
+
+/**
+ * Add all the family of <b>node</b>, including <b>node</b> itself, to
+ * the smartlist <b>sl</b>.
+ *
+ * This is used to make sure we don't pick siblings in a single path, or
+ * pick more than one relay from a family for our entry guard list.
+ * Note that a node may be added to <b>sl</b> more than once if it is
+ * part of <b>node</b>'s family for more than one reason.
+ */
+void
+nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
+{
+  const smartlist_t *all_nodes = nodelist_get_list();
+  const smartlist_t *declared_family;
+  const or_options_t *options = get_options();
+
+  tor_assert(node);
+
+  declared_family = node_get_declared_family(node);
+
+  /* Let's make sure that we have the node itself, if it's a real node. */
+  {
+    const node_t *real_node = node_get_by_id(node->identity);
+    if (real_node)
+      smartlist_add(sl, (node_t*)real_node);
+  }
+
+  /* First, add any nodes with similar network addresses. */
+  if (options->EnforceDistinctSubnets) {
+    tor_addr_t node_addr;
+    node_get_addr(node, &node_addr);
+
+    SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) {
+      tor_addr_t a;
+      node_get_addr(node2, &a);
+      if (addrs_in_same_network_family(&a, &node_addr))
+        smartlist_add(sl, (void*)node2);
+    } SMARTLIST_FOREACH_END(node2);
+  }
+
+  /* Now, add all nodes in the declared_family of this node, if they
+   * also declare this node to be in their family. */
+  if (declared_family) {
+    /* Add every r such that router declares familyness with node, and node
+     * declares familyhood with router. */
+    SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
+      const node_t *node2;
+      const smartlist_t *family2;
+      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
+        continue;
+      if (!(family2 = node_get_declared_family(node2)))
+        continue;
+      SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) {
+          if (node_nickname_matches(node, name2)) {
+            smartlist_add(sl, (void*)node2);
+            break;
+          }
+      } SMARTLIST_FOREACH_END(name2);
+    } SMARTLIST_FOREACH_END(name);
+  }
+
+  /* If the user declared any families locally, honor those too. */
+  if (options->NodeFamilySets) {
+    SMARTLIST_FOREACH(options->NodeFamilySets, const routerset_t *, rs, {
+      if (routerset_contains_node(rs, node)) {
+        routerset_get_all_nodes(sl, rs, NULL, 0);
+      }
+    });
+  }
+}
+
+/** Find a router that's up, that has this IP address, and
+ * that allows exit to this address:port, or return NULL if there
+ * isn't a good one.
+ * Don't exit enclave to excluded relays -- it wouldn't actually
+ * hurt anything, but this way there are fewer confused users.
+ */
+const node_t *
+router_find_exact_exit_enclave(const char *address, uint16_t port)
+{/*XXXX MOVE*/
+  uint32_t addr;
+  struct in_addr in;
+  tor_addr_t a;
+  const or_options_t *options = get_options();
+
+  if (!tor_inet_aton(address, &in))
+    return NULL; /* it's not an IP already */
+  addr = ntohl(in.s_addr);
+
+  tor_addr_from_ipv4h(&a, addr);
+
+  SMARTLIST_FOREACH(nodelist_get_list(), const node_t *, node, {
+    if (node_get_addr_ipv4h(node) == addr &&
+        node->is_running &&
+        compare_tor_addr_to_node_policy(&a, port, node) ==
+          ADDR_POLICY_ACCEPTED &&
+        !routerset_contains_node(options->ExcludeExitNodesUnion_, node))
+      return node;
+  });
+  return NULL;
+}
+
+/** Return 1 if <b>router</b> is not suitable for these parameters, else 0.
+ * If <b>need_uptime</b> is non-zero, we require a minimum uptime.
+ * If <b>need_capacity</b> is non-zero, we require a minimum advertised
+ * bandwidth.
+ * If <b>need_guard</b>, we require that the router is a possible entry guard.
+ */
+int
+node_is_unreliable(const node_t *node, int need_uptime,
+                   int need_capacity, int need_guard)
+{
+  if (need_uptime && !node->is_stable)
+    return 1;
+  if (need_capacity && !node->is_fast)
+    return 1;
+  if (need_guard && !node->is_possible_guard)
+    return 1;
+  return 0;
+}
+
+/** Return 1 if all running sufficiently-stable routers we can use will reject
+ * addr:port. Return 0 if any might accept it. */
+int
+router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port,
+                                    int need_uptime)
+{
+  addr_policy_result_t r;
+
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    if (node->is_running &&
+        !node_is_unreliable(node, need_uptime, 0, 0)) {
+
+      r = compare_tor_addr_to_node_policy(addr, port, node);
+
+      if (r != ADDR_POLICY_REJECTED && r != ADDR_POLICY_PROBABLY_REJECTED)
+        return 0; /* this one could be ok. good enough. */
+    }
+  } SMARTLIST_FOREACH_END(node);
+  return 1; /* all will reject. */
+}
+
+/** Mark the router with ID <b>digest</b> as running or non-running
+ * in our routerlist. */
+void
+router_set_status(const char *digest, int up)
+{
+  node_t *node;
+  tor_assert(digest);
+
+  SMARTLIST_FOREACH(router_get_fallback_dir_servers(),
+                    dir_server_t *, d,
+                    if (tor_memeq(d->digest, digest, DIGEST_LEN))
+                      d->is_running = up);
+
+  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
+                    dir_server_t *, d,
+                    if (tor_memeq(d->digest, digest, DIGEST_LEN))
+                      d->is_running = up);
+
+  node = node_get_mutable_by_id(digest);
+  if (node) {
+#if 0
+    log_debug(LD_DIR,"Marking router %s as %s.",
+              node_describe(node), up ? "up" : "down");
+#endif
+    if (!up && node_is_me(node) && !net_is_disabled())
+      log_warn(LD_NET, "We just marked ourself as down. Are your external "
+               "addresses reachable?");
+
+    if (bool_neq(node->is_running, up))
+      router_dir_info_changed();
+
+    node->is_running = up;
+  }
+}
+
+/** True iff, the last time we checked whether we had enough directory info
+ * to build circuits, the answer was "yes". If there are no exits in the
+ * consensus, we act as if we have 100% of the exit directory info. */
+static int have_min_dir_info = 0;
+
+/** Does the consensus contain nodes that can exit? */
+static consensus_path_type_t have_consensus_path = CONSENSUS_PATH_UNKNOWN;
+
+/** True iff enough has changed since the last time we checked whether we had
+ * enough directory info to build circuits that our old answer can no longer
+ * be trusted. */
+static int need_to_update_have_min_dir_info = 1;
+/** String describing what we're missing before we have enough directory
+ * info. */
+static char dir_info_status[512] = "";
+
+/** Return true iff we have enough consensus information to
+ * start building circuits.  Right now, this means "a consensus that's
+ * less than a day old, and at least 60% of router descriptors (configurable),
+ * weighted by bandwidth. Treat the exit fraction as 100% if there are
+ * no exits in the consensus."
+ * To obtain the final weighted bandwidth, we multiply the
+ * weighted bandwidth fraction for each position (guard, middle, exit). */
+MOCK_IMPL(int,
+router_have_minimum_dir_info,(void))
+{
+  static int logged_delay=0;
+  const char *delay_fetches_msg = NULL;
+  if (should_delay_dir_fetches(get_options(), &delay_fetches_msg)) {
+    if (!logged_delay)
+      log_notice(LD_DIR, "Delaying directory fetches: %s", delay_fetches_msg);
+    logged_delay=1;
+    strlcpy(dir_info_status, delay_fetches_msg,  sizeof(dir_info_status));
+    return 0;
+  }
+  logged_delay = 0; /* reset it if we get this far */
+
+  if (PREDICT_UNLIKELY(need_to_update_have_min_dir_info)) {
+    update_router_have_minimum_dir_info();
+  }
+
+  return have_min_dir_info;
+}
+
+/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node
+ * in the consensus. We update this flag in compute_frac_paths_available if
+ * there is at least one relay that has an Exit flag in the consensus.
+ * Used to avoid building exit circuits when they will almost certainly fail.
+ * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus.
+ * (This situation typically occurs during bootstrap of a test network.)
+ * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have
+ * reason to believe our last known value was invalid or has expired.
+ * If we're in a network with TestingDirAuthVoteExit set,
+ * this can cause router_have_consensus_path() to be set to
+ * CONSENSUS_PATH_EXIT, even if there are no nodes with accept exit policies.
+ */
+MOCK_IMPL(consensus_path_type_t,
+router_have_consensus_path, (void))
+{
+  return have_consensus_path;
+}
+
+/** Called when our internal view of the directory has changed.  This can be
+ * when the authorities change, networkstatuses change, the list of routerdescs
+ * changes, or number of running routers changes.
+ */
+void
+router_dir_info_changed(void)
+{
+  need_to_update_have_min_dir_info = 1;
+  rend_hsdir_routers_changed();
+  hs_service_dir_info_changed();
+  hs_client_dir_info_changed();
+}
+
+/** Return a string describing what we're missing before we have enough
+ * directory info. */
+const char *
+get_dir_info_status_string(void)
+{
+  return dir_info_status;
+}
+
+/** Iterate over the servers listed in <b>consensus</b>, and count how many of
+ * them seem like ones we'd use (store this in *<b>num_usable</b>), and how
+ * many of <em>those</em> we have descriptors for (store this in
+ * *<b>num_present</b>).
+ *
+ * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>.
+ * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes
+ * with the Exit flag.
+ * If *<b>descs_out</b> is present, add a node_t for each usable descriptor
+ * to it.
+ */
+static void
+count_usable_descriptors(int *num_present, int *num_usable,
+                         smartlist_t *descs_out,
+                         const networkstatus_t *consensus,
+                         time_t now,
+                         routerset_t *in_set,
+                         usable_descriptor_t exit_only)
+{
+  const int md = (consensus->flavor == FLAV_MICRODESC);
+  *num_present = 0, *num_usable = 0;
+
+  SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, routerstatus_t *, rs)
+    {
+       const node_t *node = node_get_by_id(rs->identity_digest);
+       if (!node)
+         continue; /* This would be a bug: every entry in the consensus is
+                    * supposed to have a node. */
+       if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit)
+         continue;
+       if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1))
+         continue;
+       if (client_would_use_router(rs, now)) {
+         const char * const digest = rs->descriptor_digest;
+         int present;
+         ++*num_usable; /* the consensus says we want it. */
+         if (md)
+           present = NULL != microdesc_cache_lookup_by_digest256(NULL, digest);
+         else
+           present = NULL != router_get_by_descriptor_digest(digest);
+         if (present) {
+           /* we have the descriptor listed in the consensus. */
+           ++*num_present;
+         }
+         if (descs_out)
+           smartlist_add(descs_out, (node_t*)node);
+       }
+     }
+  SMARTLIST_FOREACH_END(rs);
+
+  log_debug(LD_DIR, "%d usable, %d present (%s%s).",
+            *num_usable, *num_present,
+            md ? "microdesc" : "desc",
+            exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s");
+}
+
+/** Return an estimate of which fraction of usable paths through the Tor
+ * network we have available for use.  Count how many routers seem like ones
+ * we'd use (store this in *<b>num_usable_out</b>), and how many of
+ * <em>those</em> we have descriptors for (store this in
+ * *<b>num_present_out</b>.)
+ *
+ * If **<b>status_out</b> is present, allocate a new string and print the
+ * available percentages of guard, middle, and exit nodes to it, noting
+ * whether there are exits in the consensus.
+ * If there are no exits in the consensus, we treat the exit fraction as 100%,
+ * but set router_have_consensus_path() so that we can only build internal
+ * paths. */
+static double
+compute_frac_paths_available(const networkstatus_t *consensus,
+                             const or_options_t *options, time_t now,
+                             int *num_present_out, int *num_usable_out,
+                             char **status_out)
+{
+  smartlist_t *guards = smartlist_new();
+  smartlist_t *mid    = smartlist_new();
+  smartlist_t *exits  = smartlist_new();
+  double f_guard, f_mid, f_exit;
+  double f_path = 0.0;
+  /* Used to determine whether there are any exits in the consensus */
+  int np = 0;
+  /* Used to determine whether there are any exits with descriptors */
+  int nu = 0;
+  const int authdir = authdir_mode_v3(options);
+
+  count_usable_descriptors(num_present_out, num_usable_out,
+                           mid, consensus, now, NULL,
+                           USABLE_DESCRIPTOR_ALL);
+  if (options->EntryNodes) {
+    count_usable_descriptors(&np, &nu, guards, consensus, now,
+                             options->EntryNodes, USABLE_DESCRIPTOR_ALL);
+  } else {
+    SMARTLIST_FOREACH(mid, const node_t *, node, {
+      if (authdir) {
+        if (node->rs && node->rs->is_possible_guard)
+          smartlist_add(guards, (node_t*)node);
+      } else {
+        if (node->is_possible_guard)
+          smartlist_add(guards, (node_t*)node);
+      }
+    });
+  }
+
+  /* All nodes with exit flag
+   * If we're in a network with TestingDirAuthVoteExit set,
+   * this can cause false positives on have_consensus_path,
+   * incorrectly setting it to CONSENSUS_PATH_EXIT. This is
+   * an unavoidable feature of forcing authorities to declare
+   * certain nodes as exits.
+   */
+  count_usable_descriptors(&np, &nu, exits, consensus, now,
+                           NULL, USABLE_DESCRIPTOR_EXIT_ONLY);
+  log_debug(LD_NET,
+            "%s: %d present, %d usable",
+            "exits",
+            np,
+            nu);
+
+  /* We need at least 1 exit present in the consensus to consider
+   * building exit paths */
+  /* Update our understanding of whether the consensus has exits */
+  consensus_path_type_t old_have_consensus_path = have_consensus_path;
+  have_consensus_path = ((nu > 0) ?
+                         CONSENSUS_PATH_EXIT :
+                         CONSENSUS_PATH_INTERNAL);
+
+  if (have_consensus_path == CONSENSUS_PATH_INTERNAL
+      && old_have_consensus_path != have_consensus_path) {
+    log_notice(LD_NET,
+               "The current consensus has no exit nodes. "
+               "Tor can only build internal paths, "
+               "such as paths to hidden services.");
+
+    /* However, exit nodes can reachability self-test using this consensus,
+     * join the network, and appear in a later consensus. This will allow
+     * the network to build exit paths, such as paths for world wide web
+     * browsing (as distinct from hidden service web browsing). */
+  }
+
+  f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD, 1);
+  f_mid   = frac_nodes_with_descriptors(mid,    WEIGHT_FOR_MID,   0);
+  f_exit  = frac_nodes_with_descriptors(exits,  WEIGHT_FOR_EXIT,  0);
+
+  /* If we are using bridges and have at least one bridge with a full
+   * descriptor, assume f_guard is 1.0. */
+  if (options->UseBridges && num_bridges_usable(0) > 0)
+    f_guard = 1.0;
+
+  log_debug(LD_NET,
+            "f_guard: %.2f, f_mid: %.2f, f_exit: %.2f",
+             f_guard,
+             f_mid,
+             f_exit);
+
+  smartlist_free(guards);
+  smartlist_free(mid);
+  smartlist_free(exits);
+
+  if (options->ExitNodes) {
+    double f_myexit, f_myexit_unflagged;
+    smartlist_t *myexits= smartlist_new();
+    smartlist_t *myexits_unflagged = smartlist_new();
+
+    /* All nodes with exit flag in ExitNodes option */
+    count_usable_descriptors(&np, &nu, myexits, consensus, now,
+                             options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY);
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "myexits",
+              np,
+              nu);
+
+    /* Now compute the nodes in the ExitNodes option where which we don't know
+     * what their exit policy is, or we know it permits something. */
+    count_usable_descriptors(&np, &nu, myexits_unflagged,
+                             consensus, now,
+                             options->ExitNodes, USABLE_DESCRIPTOR_ALL);
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "myexits_unflagged (initial)",
+              np,
+              nu);
+
+    SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) {
+      if (node_has_preferred_descriptor(node, 0) &&
+          node_exit_policy_rejects_all(node)) {
+        SMARTLIST_DEL_CURRENT(myexits_unflagged, node);
+        /* this node is not actually an exit */
+        np--;
+        /* this node is unusable as an exit */
+        nu--;
+      }
+    } SMARTLIST_FOREACH_END(node);
+
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "myexits_unflagged (final)",
+              np,
+              nu);
+
+    f_myexit= frac_nodes_with_descriptors(myexits, WEIGHT_FOR_EXIT, 0);
+    f_myexit_unflagged=
+              frac_nodes_with_descriptors(myexits_unflagged,
+                                          WEIGHT_FOR_EXIT, 0);
+
+    log_debug(LD_NET,
+              "f_exit: %.2f, f_myexit: %.2f, f_myexit_unflagged: %.2f",
+              f_exit,
+              f_myexit,
+              f_myexit_unflagged);
+
+    /* If our ExitNodes list has eliminated every possible Exit node, and there
+     * were some possible Exit nodes, then instead consider nodes that permit
+     * exiting to some ports. */
+    if (smartlist_len(myexits) == 0 &&
+        smartlist_len(myexits_unflagged)) {
+      f_myexit = f_myexit_unflagged;
+    }
+
+    smartlist_free(myexits);
+    smartlist_free(myexits_unflagged);
+
+    /* This is a tricky point here: we don't want to make it easy for a
+     * directory to trickle exits to us until it learns which exits we have
+     * configured, so require that we have a threshold both of total exits
+     * and usable exits. */
+    if (f_myexit < f_exit)
+      f_exit = f_myexit;
+  }
+
+  /* if the consensus has no exits, treat the exit fraction as 100% */
+  if (router_have_consensus_path() != CONSENSUS_PATH_EXIT) {
+    f_exit = 1.0;
+  }
+
+  f_path = f_guard * f_mid * f_exit;
+
+  if (status_out)
+    tor_asprintf(status_out,
+                 "%d%% of guards bw, "
+                 "%d%% of midpoint bw, and "
+                 "%d%% of exit bw%s = "
+                 "%d%% of path bw",
+                 (int)(f_guard*100),
+                 (int)(f_mid*100),
+                 (int)(f_exit*100),
+                 (router_have_consensus_path() == CONSENSUS_PATH_EXIT ?
+                  "" :
+                  " (no exits in consensus)"),
+                 (int)(f_path*100));
+
+  return f_path;
+}
+
+/** We just fetched a new set of descriptors. Compute how far through
+ * the "loading descriptors" bootstrapping phase we are, so we can inform
+ * the controller of our progress. */
+int
+count_loading_descriptors_progress(void)
+{
+  int num_present = 0, num_usable=0;
+  time_t now = time(NULL);
+  const or_options_t *options = get_options();
+  const networkstatus_t *consensus =
+    networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor());
+  double paths, fraction;
+
+  if (!consensus)
+    return 0; /* can't count descriptors if we have no list of them */
+
+  paths = compute_frac_paths_available(consensus, options, now,
+                                       &num_present, &num_usable,
+                                       NULL);
+
+  fraction = paths / get_frac_paths_needed_for_circs(options,consensus);
+  if (fraction > 1.0)
+    return 0; /* it's not the number of descriptors holding us back */
+  return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int)
+    (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 -
+               BOOTSTRAP_STATUS_LOADING_DESCRIPTORS));
+}
+
+/** Return the fraction of paths needed before we're willing to build
+ * circuits, as configured in <b>options</b>, or in the consensus <b>ns</b>. */
+static double
+get_frac_paths_needed_for_circs(const or_options_t *options,
+                                const networkstatus_t *ns)
+{
+#define DFLT_PCT_USABLE_NEEDED 60
+  if (options->PathsNeededToBuildCircuits >= 0.0) {
+    return options->PathsNeededToBuildCircuits;
+  } else {
+    return networkstatus_get_param(ns, "min_paths_for_circs_pct",
+                                   DFLT_PCT_USABLE_NEEDED,
+                                   25, 95)/100.0;
+  }
+}
+
+/** Change the value of have_min_dir_info, setting it true iff we have enough
+ * network and router information to build circuits.  Clear the value of
+ * need_to_update_have_min_dir_info. */
+static void
+update_router_have_minimum_dir_info(void)
+{
+  time_t now = time(NULL);
+  int res;
+  int num_present=0, num_usable=0;
+  const or_options_t *options = get_options();
+  const networkstatus_t *consensus =
+    networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor());
+  int using_md;
+
+  if (!consensus) {
+    if (!networkstatus_get_latest_consensus())
+      strlcpy(dir_info_status, "We have no usable consensus.",
+              sizeof(dir_info_status));
+    else
+      strlcpy(dir_info_status, "We have no recent usable consensus.",
+              sizeof(dir_info_status));
+    res = 0;
+    goto done;
+  }
+
+  using_md = consensus->flavor == FLAV_MICRODESC;
+
+  /* Check fraction of available paths */
+  {
+    char *status = NULL;
+    double paths = compute_frac_paths_available(consensus, options, now,
+                                                &num_present, &num_usable,
+                                                &status);
+
+    if (paths < get_frac_paths_needed_for_circs(options,consensus)) {
+      tor_snprintf(dir_info_status, sizeof(dir_info_status),
+                   "We need more %sdescriptors: we have %d/%d, and "
+                   "can only build %d%% of likely paths. (We have %s.)",
+                   using_md?"micro":"", num_present, num_usable,
+                   (int)(paths*100), status);
+      tor_free(status);
+      res = 0;
+      control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
+      goto done;
+    }
+
+    tor_free(status);
+    res = 1;
+  }
+
+  { /* Check entry guard dirinfo status */
+    char *guard_error = entry_guards_get_err_str_if_dir_info_missing(using_md,
+                                                             num_present,
+                                                             num_usable);
+    if (guard_error) {
+      strlcpy(dir_info_status, guard_error, sizeof(dir_info_status));
+      tor_free(guard_error);
+      res = 0;
+      goto done;
+    }
+  }
+
+ done:
+
+  /* If paths have just become available in this update. */
+  if (res && !have_min_dir_info) {
+    control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO");
+    if (control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0) == 0) {
+      log_notice(LD_DIR,
+              "We now have enough directory information to build circuits.");
+    }
+  }
+
+  /* If paths have just become unavailable in this update. */
+  if (!res && have_min_dir_info) {
+    int quiet = directory_too_idle_to_fetch_descriptors(options, now);
+    tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR,
+        "Our directory information is no longer up-to-date "
+        "enough to build circuits: %s", dir_info_status);
+
+    /* a) make us log when we next complete a circuit, so we know when Tor
+     * is back up and usable, and b) disable some activities that Tor
+     * should only do while circuits are working, like reachability tests
+     * and fetching bridge descriptors only over circuits. */
+    note_that_we_maybe_cant_complete_circuits();
+    have_consensus_path = CONSENSUS_PATH_UNKNOWN;
+    control_event_client_status(LOG_NOTICE, "NOT_ENOUGH_DIR_INFO");
+  }
+  have_min_dir_info = res;
+  need_to_update_have_min_dir_info = 0;
+}
+
diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h
new file mode 100644
index 0000000000..ed3a542971
--- /dev/null
+++ b/src/feature/nodelist/nodelist.h
@@ -0,0 +1,168 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodelist.h
+ * \brief Header file for nodelist.c.
+ **/
+
+#ifndef TOR_NODELIST_H
+#define TOR_NODELIST_H
+
+struct ed25519_public_key_t;
+struct curve25519_public_key_t;
+
+#define node_assert_ok(n) STMT_BEGIN {                          \
+    tor_assert((n)->ri || (n)->rs);                             \
+  } STMT_END
+
+MOCK_DECL(node_t *, node_get_mutable_by_id,(const char *identity_digest));
+MOCK_DECL(const node_t *, node_get_by_id, (const char *identity_digest));
+node_t *node_get_mutable_by_ed25519_id(
+                            const struct ed25519_public_key_t *ed_id);
+MOCK_DECL(const node_t *, node_get_by_ed25519_id,
+          (const struct ed25519_public_key_t *ed_id));
+
+#define NNF_NO_WARN_UNNAMED (1u<<0)
+
+const node_t *node_get_by_hex_id(const char *identity_digest,
+                                 unsigned flags);
+node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out);
+node_t *nodelist_add_microdesc(microdesc_t *md);
+void nodelist_set_consensus(networkstatus_t *ns);
+void nodelist_ensure_freshness(networkstatus_t *ns);
+int nodelist_probably_contains_address(const tor_addr_t *addr);
+
+void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md);
+void nodelist_remove_routerinfo(routerinfo_t *ri);
+void nodelist_purge(void);
+smartlist_t *nodelist_find_nodes_with_microdesc(const microdesc_t *md);
+
+void nodelist_free_all(void);
+void nodelist_assert_ok(void);
+
+MOCK_DECL(const node_t *, node_get_by_nickname,
+          (const char *nickname, unsigned flags));
+void node_get_verbose_nickname(const node_t *node,
+                               char *verbose_name_out);
+void node_get_verbose_nickname_by_id(const char *id_digest,
+                                char *verbose_name_out);
+int node_is_dir(const node_t *node);
+int node_is_good_exit(const node_t *node);
+int node_has_any_descriptor(const node_t *node);
+int node_has_preferred_descriptor(const node_t *node,
+                                  int for_direct_connect);
+int node_get_purpose(const node_t *node);
+#define node_is_bridge(node) \
+  (node_get_purpose((node)) == ROUTER_PURPOSE_BRIDGE)
+int node_is_me(const node_t *node);
+int node_exit_policy_rejects_all(const node_t *node);
+int node_exit_policy_is_exact(const node_t *node, sa_family_t family);
+smartlist_t *node_get_all_orports(const node_t *node);
+int node_allows_single_hop_exits(const node_t *node);
+const char *node_get_nickname(const node_t *node);
+const char *node_get_platform(const node_t *node);
+uint32_t node_get_prim_addr_ipv4h(const node_t *node);
+void node_get_address_string(const node_t *node, char *cp, size_t len);
+long node_get_declared_uptime(const node_t *node);
+const smartlist_t *node_get_declared_family(const node_t *node);
+const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
+int node_ed25519_id_matches(const node_t *node,
+                            const struct ed25519_public_key_t *id);
+int node_supports_ed25519_link_authentication(const node_t *node,
+                                              int compatible_with_us);
+int node_supports_v3_hsdir(const node_t *node);
+int node_supports_ed25519_hs_intro(const node_t *node);
+int node_supports_v3_rendezvous_point(const node_t *node);
+const uint8_t *node_get_rsa_id_digest(const node_t *node);
+
+int node_has_ipv6_addr(const node_t *node);
+int node_has_ipv6_orport(const node_t *node);
+int node_has_ipv6_dirport(const node_t *node);
+/* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */
+#define node_ipv6_preferred(node) node_ipv6_or_preferred(node)
+int node_ipv6_or_preferred(const node_t *node);
+void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out);
+int node_ipv6_dir_preferred(const node_t *node);
+void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out);
+int node_has_curve25519_onion_key(const node_t *node);
+const struct curve25519_public_key_t *node_get_curve25519_onion_key(
+                                  const node_t *node);
+
+MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
+
+/* Temporary during transition to multiple addresses.  */
+void node_get_addr(const node_t *node, tor_addr_t *addr_out);
+#define node_get_addr_ipv4h(n) node_get_prim_addr_ipv4h((n))
+
+void nodelist_refresh_countries(void);
+void node_set_country(node_t *node);
+void nodelist_add_node_and_family(smartlist_t *nodes, const node_t *node);
+int nodes_in_same_family(const node_t *node1, const node_t *node2);
+
+const node_t *router_find_exact_exit_enclave(const char *address,
+                                             uint16_t port);
+int node_is_unreliable(const node_t *router, int need_uptime,
+                         int need_capacity, int need_guard);
+int router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port,
+                                        int need_uptime);
+void router_set_status(const char *digest, int up);
+int addrs_in_same_network_family(const tor_addr_t *a1,
+                                 const tor_addr_t *a2);
+
+/** router_have_minimum_dir_info tests to see if we have enough
+ * descriptor information to create circuits.
+ * If there are exits in the consensus, we wait until we have enough
+ * info to create exit paths before creating any circuits. If there are
+ * no exits in the consensus, we wait for enough info to create internal
+ * paths, and should avoid creating exit paths, as they will simply fail.
+ * We make sure we create all available circuit types at the same time. */
+MOCK_DECL(int, router_have_minimum_dir_info,(void));
+
+/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node
+ * in the consensus. We update this flag in compute_frac_paths_available if
+ * there is at least one relay that has an Exit flag in the consensus.
+ * Used to avoid building exit circuits when they will almost certainly fail.
+ * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus.
+ * (This situation typically occurs during bootstrap of a test network.)
+ * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have
+ * reason to believe our last known value was invalid or has expired.
+ */
+typedef enum {
+  /* we haven't checked yet, or we have invalidated our previous check */
+  CONSENSUS_PATH_UNKNOWN = -1,
+  /* The consensus only has internal relays, and we should only
+   * create internal paths, circuits, streams, ... */
+  CONSENSUS_PATH_INTERNAL = 0,
+  /* The consensus has at least one exit, and can therefore (potentially)
+   * create exit and internal paths, circuits, streams, ... */
+  CONSENSUS_PATH_EXIT = 1
+} consensus_path_type_t;
+
+MOCK_DECL(consensus_path_type_t, router_have_consensus_path, (void));
+
+void router_dir_info_changed(void);
+const char *get_dir_info_status_string(void);
+int count_loading_descriptors_progress(void);
+
+#ifdef NODELIST_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(NODELIST_PRIVATE) */
+
+MOCK_DECL(int, get_estimated_address_per_node, (void));
+
+#endif /* !defined(TOR_NODELIST_H) */
diff --git a/src/feature/nodelist/parsecommon.c b/src/feature/nodelist/parsecommon.c
new file mode 100644
index 0000000000..4340f28225
--- /dev/null
+++ b/src/feature/nodelist/parsecommon.c
@@ -0,0 +1,458 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.c
+ * \brief Common code to parse and validate various type of descriptors.
+ **/
+
+#include "or/parsecommon.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/encoding/binascii.h"
+#include "lib/container/smartlist.h"
+#include "lib/string/util_string.h"
+#include "lib/string/printf.h"
+#include "lib/memarea/memarea.h"
+#include "lib/crypt_ops/crypto.h"
+
+#include <string.h>
+
+#define MIN_ANNOTATION A_PURPOSE
+#define MAX_ANNOTATION A_UNKNOWN_
+
+#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
+#define ALLOC(sz) memarea_alloc(area,sz)
+#define STRDUP(str) memarea_strdup(area,str)
+#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
+
+#define RET_ERR(msg)                                               \
+  STMT_BEGIN                                                       \
+    if (tok) token_clear(tok);                                      \
+    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
+    tok->tp = ERR_;                                                \
+    tok->error = STRDUP(msg);                                      \
+    goto done_tokenizing;                                          \
+  STMT_END
+
+/** Free all resources allocated for <b>tok</b> */
+void
+token_clear(directory_token_t *tok)
+{
+  if (tok->key)
+    crypto_pk_free(tok->key);
+}
+
+/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
+ * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
+ * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
+ * entire string.
+ */
+int
+tokenize_string(memarea_t *area,
+                const char *start, const char *end, smartlist_t *out,
+                token_rule_t *table, int flags)
+{
+  const char **s;
+  directory_token_t *tok = NULL;
+  int counts[NIL_];
+  int i;
+  int first_nonannotation;
+  int prev_len = smartlist_len(out);
+  tor_assert(area);
+
+  s = &start;
+  if (!end) {
+    end = start+strlen(start);
+  } else {
+    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
+    if (memchr(start, '\0', end-start)) {
+      log_warn(LD_DIR, "parse error: internal NUL character.");
+      return -1;
+    }
+  }
+  for (i = 0; i < NIL_; ++i)
+    counts[i] = 0;
+
+  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
+
+  while (*s < end && (!tok || tok->tp != EOF_)) {
+    tok = get_next_token(area, s, end, table);
+    if (tok->tp == ERR_) {
+      log_warn(LD_DIR, "parse error: %s", tok->error);
+      token_clear(tok);
+      return -1;
+    }
+    ++counts[tok->tp];
+    smartlist_add(out, tok);
+    *s = eat_whitespace_eos(*s, end);
+  }
+
+  if (flags & TS_NOCHECK)
+    return 0;
+
+  if ((flags & TS_ANNOTATIONS_OK)) {
+    first_nonannotation = -1;
+    for (i = 0; i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
+        first_nonannotation = i;
+        break;
+      }
+    }
+    if (first_nonannotation < 0) {
+      log_warn(LD_DIR, "parse error: item contains only annotations");
+      return -1;
+    }
+    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
+        return -1;
+      }
+    }
+    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
+      if (first_nonannotation != prev_len) {
+        log_warn(LD_DIR, "parse error: Unexpected annotations.");
+        return -1;
+      }
+    }
+  } else {
+    for (i=0;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: no annotations allowed.");
+        return -1;
+      }
+    }
+    first_nonannotation = 0;
+  }
+  for (i = 0; table[i].t; ++i) {
+    if (counts[table[i].v] < table[i].min_cnt) {
+      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
+      return -1;
+    }
+    if (counts[table[i].v] > table[i].max_cnt) {
+      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
+      return -1;
+    }
+    if (table[i].pos & AT_START) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+    if (table[i].pos & AT_END) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/** Helper: parse space-separated arguments from the string <b>s</b> ending at
+ * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
+ * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
+ * all storage in <b>area</b>.  Return the number of arguments parsed, or
+ * return -1 if there was an insanely high number of arguments. */
+static inline int
+get_token_arguments(memarea_t *area, directory_token_t *tok,
+                    const char *s, const char *eol)
+{
+/** Largest number of arguments we'll accept to any token, ever. */
+#define MAX_ARGS 512
+  char *mem = memarea_strndup(area, s, eol-s);
+  char *cp = mem;
+  int j = 0;
+  char *args[MAX_ARGS];
+  memset(args, 0, sizeof(args));
+  while (*cp) {
+    if (j == MAX_ARGS)
+      return -1;
+    args[j++] = cp;
+    cp = (char*)find_whitespace(cp);
+    if (!cp || !*cp)
+      break; /* End of the line. */
+    *cp++ = '\0';
+    cp = (char*)eat_whitespace(cp);
+  }
+  tok->n_args = j;
+  tok->args = memarea_memdup(area, args, j*sizeof(char*));
+  return j;
+#undef MAX_ARGS
+}
+
+/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
+ * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
+ * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
+ * conform to the syntax we wanted.
+ **/
+static inline directory_token_t *
+token_check_object(memarea_t *area, const char *kwd,
+                   directory_token_t *tok, obj_syntax o_syn)
+{
+  char ebuf[128];
+  switch (o_syn) {
+    case NO_OBJ:
+      /* No object is allowed for this token. */
+      if (tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_OBJ:
+      /* There must be a (non-key) object. */
+      if (!tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
+    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
+      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
+                     kwd, crypto_pk_num_bits(tok->key));
+        RET_ERR(ebuf);
+      }
+      /* fall through */
+    case NEED_KEY: /* There must be some kind of key. */
+      if (!tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (o_syn != NEED_SKEY_1024) {
+        if (crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Private key given for %s, which wants a public key", kwd);
+          RET_ERR(ebuf);
+        }
+      } else { /* o_syn == NEED_SKEY_1024 */
+        if (!crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Public key given for %s, which wants a private key", kwd);
+          RET_ERR(ebuf);
+        }
+      }
+      break;
+    case OBJ_OK:
+      /* Anything goes with this token. */
+      break;
+  }
+
+ done_tokenizing:
+  return tok;
+}
+
+/** Helper function: read the next token from *s, advance *s to the end of the
+ * token, and return the parsed token.  Parse *<b>s</b> according to the list
+ * of tokens in <b>table</b>.
+ */
+directory_token_t *
+get_next_token(memarea_t *area,
+               const char **s, const char *eos, token_rule_t *table)
+{
+  /** Reject any object at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
+  /** Reject any line at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_LINE_LENGTH (128*1024)
+
+  const char *next, *eol, *obstart;
+  size_t obname_len;
+  int i;
+  directory_token_t *tok;
+  obj_syntax o_syn = NO_OBJ;
+  char ebuf[128];
+  const char *kwd = "";
+
+  tor_assert(area);
+  tok = ALLOC_ZERO(sizeof(directory_token_t));
+  tok->tp = ERR_;
+
+  /* Set *s to first token, eol to end-of-line, next to after first token */
+  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol)
+    eol = eos;
+  if (eol - *s > MAX_LINE_LENGTH) {
+    RET_ERR("Line far too long");
+  }
+
+  next = find_whitespace_eos(*s, eol);
+
+  if (!strcmp_len(*s, "opt", next-*s)) {
+    /* Skip past an "opt" at the start of the line. */
+    *s = eat_whitespace_eos_no_nl(next, eol);
+    next = find_whitespace_eos(*s, eol);
+  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
+    RET_ERR("Unexpected EOF");
+  }
+
+  /* Search the table for the appropriate entry.  (I tried a binary search
+   * instead, but it wasn't any faster.) */
+  for (i = 0; table[i].t ; ++i) {
+    if (!strcmp_len(*s, table[i].t, next-*s)) {
+      /* We've found the keyword. */
+      kwd = table[i].t;
+      tok->tp = table[i].v;
+      o_syn = table[i].os;
+      *s = eat_whitespace_eos_no_nl(next, eol);
+      /* We go ahead whether there are arguments or not, so that tok->args is
+       * always set if we want arguments. */
+      if (table[i].concat_args) {
+        /* The keyword takes the line as a single argument */
+        tok->args = ALLOC(sizeof(char*));
+        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
+        tok->n_args = 1;
+      } else {
+        /* This keyword takes multiple arguments. */
+        if (get_token_arguments(area, tok, *s, eol)<0) {
+          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
+          RET_ERR(ebuf);
+        }
+        *s = eol;
+      }
+      if (tok->n_args < table[i].min_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
+        RET_ERR(ebuf);
+      } else if (tok->n_args > table[i].max_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    }
+  }
+
+  if (tok->tp == ERR_) {
+    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
+    if (*s < eol && **s == '@')
+      tok->tp = A_UNKNOWN_;
+    else
+      tok->tp = K_OPT;
+    tok->args = ALLOC(sizeof(char*));
+    tok->args[0] = STRNDUP(*s, eol-*s);
+    tok->n_args = 1;
+    o_syn = OBJ_OK;
+  }
+
+  /* Check whether there's an object present */
+  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
+    goto check_object;
+
+  obstart = *s; /* Set obstart to start of object spec */
+  if (*s+16 >= eol || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
+      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
+    RET_ERR("Malformed object: bad begin line");
+  }
+  tok->object_type = STRNDUP(*s+11, eol-*s-16);
+  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
+  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
+
+  /* Go to the end of the object */
+  next = tor_memstr(*s, eos-*s, "-----END ");
+  if (!next) {
+    RET_ERR("Malformed object: missing object end line");
+  }
+  tor_assert(eos >= next);
+  eol = memchr(next, '\n', eos-next);
+  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
+    eol = eos;
+  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
+  if ((size_t)(eol-next) != 9+obname_len+5 ||
+      strcmp_len(next+9, tok->object_type, obname_len) ||
+      strcmp_len(eol-5, "-----", 5)) {
+    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
+             tok->object_type);
+    ebuf[sizeof(ebuf)-1] = '\0';
+    RET_ERR(ebuf);
+  }
+  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
+    RET_ERR("Couldn't parse object: missing footer or object much too big.");
+
+  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse public key.");
+  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse private key.");
+  } else { /* If it's something else, try to base64-decode it */
+    int r;
+    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
+    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+  *s = eol;
+
+ check_object:
+  tok = token_check_object(area, kwd, tok, o_syn);
+
+ done_tokenizing:
+  return tok;
+
+#undef RET_ERR
+#undef ALLOC
+#undef ALLOC_ZERO
+#undef STRDUP
+#undef STRNDUP
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
+ * with an assert if no such keyword is found.
+ */
+directory_token_t *
+find_by_keyword_(smartlist_t *s, directory_keyword keyword,
+                 const char *keyword_as_string)
+{
+  directory_token_t *tok = find_opt_by_keyword(s, keyword);
+  if (PREDICT_UNLIKELY(!tok)) {
+    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
+         "been validated. Internal error.", keyword_as_string, (int)keyword);
+    tor_assert(tok);
+  }
+  return tok;
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
+ * NULL if no such keyword is found.
+ */
+directory_token_t *
+find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword)
+{
+  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
+  return NULL;
+}
+
+/** If there are any directory_token_t entries in <b>s</b> whose keyword is
+ * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
+ * in the same order in which they occur in <b>s</b>.  Otherwise return
+ * NULL. */
+smartlist_t *
+find_all_by_keyword(const smartlist_t *s, directory_keyword k)
+{
+  smartlist_t *out = NULL;
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+                    if (t->tp == k) {
+                    if (!out)
+                    out = smartlist_new();
+                    smartlist_add(out, t);
+                    });
+  return out;
+}
diff --git a/src/feature/nodelist/parsecommon.h b/src/feature/nodelist/parsecommon.h
new file mode 100644
index 0000000000..d0f3810c0b
--- /dev/null
+++ b/src/feature/nodelist/parsecommon.h
@@ -0,0 +1,324 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.h
+ * \brief Header file for parsecommon.c
+ **/
+
+#ifndef TOR_PARSECOMMON_H
+#define TOR_PARSECOMMON_H
+
+#include <stddef.h>
+
+struct smartlist_t;
+struct crypto_pk_t;
+struct memarea_t;
+
+/** Enumeration of possible token types.  The ones starting with K_ correspond
+* to directory 'keywords'. A_ is for an annotation, R or C is related to
+* hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
+* end-of-file marker, and NIL_ is used to encode not-a-token.
+*/
+typedef enum {
+  K_ACCEPT = 0,
+  K_ACCEPT6,
+  K_DIRECTORY_SIGNATURE,
+  K_RECOMMENDED_SOFTWARE,
+  K_REJECT,
+  K_REJECT6,
+  K_ROUTER,
+  K_SIGNED_DIRECTORY,
+  K_SIGNING_KEY,
+  K_ONION_KEY,
+  K_ONION_KEY_NTOR,
+  K_ROUTER_SIGNATURE,
+  K_PUBLISHED,
+  K_RUNNING_ROUTERS,
+  K_ROUTER_STATUS,
+  K_PLATFORM,
+  K_PROTO,
+  K_OPT,
+  K_BANDWIDTH,
+  K_CONTACT,
+  K_NETWORK_STATUS,
+  K_UPTIME,
+  K_DIR_SIGNING_KEY,
+  K_FAMILY,
+  K_FINGERPRINT,
+  K_HIBERNATING,
+  K_READ_HISTORY,
+  K_WRITE_HISTORY,
+  K_NETWORK_STATUS_VERSION,
+  K_DIR_SOURCE,
+  K_DIR_OPTIONS,
+  K_CLIENT_VERSIONS,
+  K_SERVER_VERSIONS,
+  K_RECOMMENDED_CLIENT_PROTOCOLS,
+  K_RECOMMENDED_RELAY_PROTOCOLS,
+  K_REQUIRED_CLIENT_PROTOCOLS,
+  K_REQUIRED_RELAY_PROTOCOLS,
+  K_OR_ADDRESS,
+  K_ID,
+  K_P,
+  K_P6,
+  K_R,
+  K_A,
+  K_S,
+  K_V,
+  K_W,
+  K_M,
+  K_EXTRA_INFO,
+  K_EXTRA_INFO_DIGEST,
+  K_CACHES_EXTRA_INFO,
+  K_HIDDEN_SERVICE_DIR,
+  K_ALLOW_SINGLE_HOP_EXITS,
+  K_IPV6_POLICY,
+  K_ROUTER_SIG_ED25519,
+  K_IDENTITY_ED25519,
+  K_MASTER_KEY_ED25519,
+  K_ONION_KEY_CROSSCERT,
+  K_NTOR_ONION_KEY_CROSSCERT,
+
+  K_DIRREQ_END,
+  K_DIRREQ_V2_IPS,
+  K_DIRREQ_V3_IPS,
+  K_DIRREQ_V2_REQS,
+  K_DIRREQ_V3_REQS,
+  K_DIRREQ_V2_SHARE,
+  K_DIRREQ_V3_SHARE,
+  K_DIRREQ_V2_RESP,
+  K_DIRREQ_V3_RESP,
+  K_DIRREQ_V2_DIR,
+  K_DIRREQ_V3_DIR,
+  K_DIRREQ_V2_TUN,
+  K_DIRREQ_V3_TUN,
+  K_ENTRY_END,
+  K_ENTRY_IPS,
+  K_CELL_END,
+  K_CELL_PROCESSED,
+  K_CELL_QUEUED,
+  K_CELL_TIME,
+  K_CELL_CIRCS,
+  K_EXIT_END,
+  K_EXIT_WRITTEN,
+  K_EXIT_READ,
+  K_EXIT_OPENED,
+
+  K_DIR_KEY_CERTIFICATE_VERSION,
+  K_DIR_IDENTITY_KEY,
+  K_DIR_KEY_PUBLISHED,
+  K_DIR_KEY_EXPIRES,
+  K_DIR_KEY_CERTIFICATION,
+  K_DIR_KEY_CROSSCERT,
+  K_DIR_ADDRESS,
+  K_DIR_TUNNELLED,
+
+  K_VOTE_STATUS,
+  K_VALID_AFTER,
+  K_FRESH_UNTIL,
+  K_VALID_UNTIL,
+  K_VOTING_DELAY,
+
+  K_KNOWN_FLAGS,
+  K_PARAMS,
+  K_BW_WEIGHTS,
+  K_VOTE_DIGEST,
+  K_CONSENSUS_DIGEST,
+  K_ADDITIONAL_DIGEST,
+  K_ADDITIONAL_SIGNATURE,
+  K_CONSENSUS_METHODS,
+  K_CONSENSUS_METHOD,
+  K_LEGACY_DIR_KEY,
+  K_DIRECTORY_FOOTER,
+  K_SIGNING_CERT_ED,
+  K_SR_FLAG,
+  K_COMMIT,
+  K_PREVIOUS_SRV,
+  K_CURRENT_SRV,
+  K_PACKAGE,
+
+  A_PURPOSE,
+  A_LAST_LISTED,
+  A_UNKNOWN_,
+
+  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+  R_VERSION,
+  R_PERMANENT_KEY,
+  R_SECRET_ID_PART,
+  R_PUBLICATION_TIME,
+  R_PROTOCOL_VERSIONS,
+  R_INTRODUCTION_POINTS,
+  R_SIGNATURE,
+
+  R_HS_DESCRIPTOR, /* From version 3, this MUST be generic to all future
+                      descriptor versions thus making it R_. */
+  R3_DESC_LIFETIME,
+  R3_DESC_SIGNING_CERT,
+  R3_REVISION_COUNTER,
+  R3_SUPERENCRYPTED,
+  R3_SIGNATURE,
+  R3_CREATE2_FORMATS,
+  R3_INTRO_AUTH_REQUIRED,
+  R3_SINGLE_ONION_SERVICE,
+  R3_INTRODUCTION_POINT,
+  R3_INTRO_ONION_KEY,
+  R3_INTRO_AUTH_KEY,
+  R3_INTRO_ENC_KEY,
+  R3_INTRO_ENC_KEY_CERT,
+  R3_INTRO_LEGACY_KEY,
+  R3_INTRO_LEGACY_KEY_CERT,
+  R3_DESC_AUTH_TYPE,
+  R3_DESC_AUTH_KEY,
+  R3_DESC_AUTH_CLIENT,
+  R3_ENCRYPTED,
+
+  R_IPO_IDENTIFIER,
+  R_IPO_IP_ADDRESS,
+  R_IPO_ONION_PORT,
+  R_IPO_ONION_KEY,
+  R_IPO_SERVICE_KEY,
+
+  C_CLIENT_NAME,
+  C_DESCRIPTOR_COOKIE,
+  C_CLIENT_KEY,
+
+  ERR_,
+  EOF_,
+  NIL_
+} directory_keyword;
+
+/** Structure to hold a single directory token.
+ *
+ * We parse a directory by breaking it into "tokens", each consisting
+ * of a keyword, a line full of arguments, and a binary object.  The
+ * arguments and object are both optional, depending on the keyword
+ * type.
+ *
+ * This structure is only allocated in memareas; do not allocate it on
+ * the heap, or token_clear() won't work.
+ */
+typedef struct directory_token_t {
+  directory_keyword tp;        /**< Type of the token. */
+  int n_args:30;               /**< Number of elements in args */
+  char **args;                 /**< Array of arguments from keyword line. */
+
+  char *object_type;           /**< -----BEGIN [object_type]-----*/
+  size_t object_size;          /**< Bytes in object_body */
+  char *object_body;           /**< Contents of object, base64-decoded. */
+
+  struct crypto_pk_t *key;     /**< For public keys only.  Heap-allocated. */
+
+  char *error;                 /**< For ERR_ tokens only. */
+} directory_token_t;
+
+/** We use a table of rules to decide how to parse each token type. */
+
+/** Rules for whether the keyword needs an object. */
+typedef enum {
+  NO_OBJ,        /**< No object, ever. */
+  NEED_OBJ,      /**< Object is required. */
+  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
+  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
+  NEED_KEY,      /**< Object is required, and must be a public key. */
+  OBJ_OK,        /**< Object is optional. */
+} obj_syntax;
+
+#define AT_START 1
+#define AT_END 2
+
+#define TS_ANNOTATIONS_OK 1
+#define TS_NOCHECK 2
+#define TS_NO_NEW_ANNOTATIONS 4
+
+/**
+ * @name macros for defining token rules
+ *
+ * Helper macros to define token tables.  's' is a string, 't' is a
+ * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
+ * object syntax.
+ */
+/**@{*/
+
+/** Appears to indicate the end of a table. */
+#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions: used for obsolete document types */
+#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions on multiplicity or location. */
+#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item that must appear exactly once */
+#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
+/** An item that must appear exactly once, at the start of the document */
+#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
+/** An item that must appear exactly once, at the end of the document */
+#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
+/** An item that must appear one or more times */
+#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
+/** An item that must appear no more than once */
+#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
+/** An annotation that must appear no more than once */
+#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
+
+/** Argument multiplicity: any number of arguments. */
+#define ARGS        0,INT_MAX,0
+/** Argument multiplicity: no arguments. */
+#define NO_ARGS     0,0,0
+/** Argument multiplicity: concatenate all arguments. */
+#define CONCAT_ARGS 1,1,1
+/** Argument multiplicity: at least <b>n</b> arguments. */
+#define GE(n)       n,INT_MAX,0
+/** Argument multiplicity: exactly <b>n</b> arguments. */
+#define EQ(n)       n,n,0
+/**@}*/
+
+/** Determines the parsing rules for a single token type. */
+typedef struct token_rule_t {
+  /** The string value of the keyword identifying the type of item. */
+  const char *t;
+  /** The corresponding directory_keyword enum. */
+  directory_keyword v;
+  /** Minimum number of arguments for this item */
+  int min_args;
+  /** Maximum number of arguments for this item */
+  int max_args;
+  /** If true, we concatenate all arguments for this item into a single
+   * string. */
+  int concat_args;
+  /** Requirements on object syntax for this item. */
+  obj_syntax os;
+  /** Lowest number of times this item may appear in a document. */
+  int min_cnt;
+  /** Highest number of times this item may appear in a document. */
+  int max_cnt;
+  /** One or more of AT_START/AT_END to limit where the item may appear in a
+   * document. */
+  int pos;
+  /** True iff this token is an annotation. */
+  int is_annotation;
+} token_rule_t;
+
+void token_clear(directory_token_t *tok);
+
+int tokenize_string(struct memarea_t *area,
+                    const char *start, const char *end,
+                    struct smartlist_t *out,
+                    token_rule_t *table,
+                    int flags);
+directory_token_t *get_next_token(struct memarea_t *area,
+                                  const char **s,
+                                  const char *eos,
+                                  token_rule_t *table);
+
+directory_token_t *find_by_keyword_(struct smartlist_t *s,
+                                    directory_keyword keyword,
+                                    const char *keyword_str);
+
+#define find_by_keyword(s, keyword) \
+  find_by_keyword_((s), (keyword), #keyword)
+
+directory_token_t *find_opt_by_keyword(const struct smartlist_t *s,
+                                       directory_keyword keyword);
+struct smartlist_t * find_all_by_keyword(const struct smartlist_t *s,
+                                         directory_keyword k);
+
+#endif /* !defined(TOR_PARSECOMMON_H) */
diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h
new file mode 100644
index 0000000000..89a7702b30
--- /dev/null
+++ b/src/feature/nodelist/routerinfo_st.h
@@ -0,0 +1,108 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERINFO_ST_H
+#define ROUTERINFO_ST_H
+
+#include "or/signed_descriptor_st.h"
+
+struct curve25519_public_key_t;
+
+/** Information about another onion router in the network. */
+struct routerinfo_t {
+  signed_descriptor_t cache_info;
+  char *nickname; /**< Human-readable OR name. */
+
+  uint32_t addr; /**< IPv4 address of OR, in host order. */
+  uint16_t or_port; /**< Port for TLS connections. */
+  uint16_t dir_port; /**< Port for HTTP directory connections. */
+
+  /** A router's IPv6 address, if it has one. */
+  /* XXXXX187 Actually these should probably be part of a list of addresses,
+   * not just a special case.  Use abstractions to access these; don't do it
+   * directly. */
+  tor_addr_t ipv6_addr;
+  uint16_t ipv6_orport;
+
+  crypto_pk_t *onion_pkey; /**< Public RSA key for onions. */
+  crypto_pk_t *identity_pkey;  /**< Public RSA key for signing. */
+  /** Public curve25519 key for onions */
+  struct curve25519_public_key_t *onion_curve25519_pkey;
+  /** What's the earliest expiration time on all the certs in this
+   * routerinfo? */
+  time_t cert_expiration_time;
+
+  char *platform; /**< What software/operating system is this OR using? */
+
+  char *protocol_list; /**< Encoded list of subprotocol versions supported
+                        * by this OR */
+
+  /* link info */
+  uint32_t bandwidthrate; /**< How many bytes does this OR add to its token
+                           * bucket per second? */
+  uint32_t bandwidthburst; /**< How large is this OR's token bucket? */
+  /** How many bytes/s is this router known to handle? */
+  uint32_t bandwidthcapacity;
+  smartlist_t *exit_policy; /**< What streams will this OR permit
+                             * to exit on IPv4?  NULL for 'reject *:*'. */
+  /** What streams will this OR permit to exit on IPv6?
+   * NULL for 'reject *:*' */
+  struct short_policy_t *ipv6_exit_policy;
+  long uptime; /**< How many seconds the router claims to have been up */
+  smartlist_t *declared_family; /**< Nicknames of router which this router
+                                 * claims are its family. */
+  char *contact_info; /**< Declared contact info for this router. */
+  unsigned int is_hibernating:1; /**< Whether the router claims to be
+                                  * hibernating */
+  unsigned int caches_extra_info:1; /**< Whether the router says it caches and
+                                     * serves extrainfo documents. */
+  unsigned int allow_single_hop_exits:1;  /**< Whether the router says
+                                           * it allows single hop exits. */
+
+  unsigned int wants_to_be_hs_dir:1; /**< True iff this router claims to be
+                                      * a hidden service directory. */
+  unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this
+                                         * router rejects everything. */
+  /** True if, after we have added this router, we should re-launch
+   * tests for it. */
+  unsigned int needs_retest_if_added:1;
+
+  /** True iff this router included "tunnelled-dir-server" in its descriptor,
+   * implying it accepts tunnelled directory requests, or it advertised
+   * dir_port > 0. */
+  unsigned int supports_tunnelled_dir_requests:1;
+
+  /** Used during voting to indicate that we should not include an entry for
+   * this routerinfo. Used only during voting. */
+  unsigned int omit_from_vote:1;
+
+  /** Flags to summarize the protocol versions for this routerinfo_t. */
+  protover_summary_flags_t pv;
+
+/** Tor can use this router for general positions in circuits; we got it
+ * from a directory server as usual, or we're an authority and a server
+ * uploaded it. */
+#define ROUTER_PURPOSE_GENERAL 0
+/** Tor should avoid using this router for circuit-building: we got it
+ * from a controller.  If the controller wants to use it, it'll have to
+ * ask for it by identity. */
+#define ROUTER_PURPOSE_CONTROLLER 1
+/** Tor should use this router only for bridge positions in circuits: we got
+ * it via a directory request from the bridge itself, or a bridge
+ * authority. */
+#define ROUTER_PURPOSE_BRIDGE 2
+/** Tor should not use this router; it was marked in cached-descriptors with
+ * a purpose we didn't recognize. */
+#define ROUTER_PURPOSE_UNKNOWN 255
+
+  /** In what way did we find out about this router?  One of ROUTER_PURPOSE_*.
+   * Routers of different purposes are kept segregated and used for different
+   * things; see notes on ROUTER_PURPOSE_* macros above.
+   */
+  uint8_t purpose;
+};
+
+#endif
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
new file mode 100644
index 0000000000..76a236ff20
--- /dev/null
+++ b/src/feature/nodelist/routerlist.c
@@ -0,0 +1,5848 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerlist.c
+ * \brief Code to
+ * maintain and access the global list of routerinfos for known
+ * servers.
+ *
+ * A "routerinfo_t" object represents a single self-signed router
+ * descriptor, as generated by a Tor relay in order to tell the rest of
+ * the world about its keys, address, and capabilities.  An
+ * "extrainfo_t" object represents an adjunct "extra-info" object,
+ * certified by a corresponding router descriptor, reporting more
+ * information about the relay that nearly all users will not need.
+ *
+ * Most users will not use router descriptors for most relays.  Instead,
+ * they use the information in microdescriptors and in the consensus
+ * networkstatus.
+ *
+ * Right now, routerinfo_t objects are used in these ways:
+ *  <ul>
+ *   <li>By clients, in order to learn about bridge keys and capabilities.
+ *     (Bridges aren't listed in the consensus networkstatus, so they
+ *     can't have microdescriptors.)
+ *   <li>By relays, since relays want more information about other relays
+ *     than they can learn from microdescriptors. (TODO: Is this still true?)
+ *   <li>By authorities, which receive them and use them to generate the
+ *     consensus and the microdescriptors.
+ *   <li>By all directory caches, which download them in case somebody
+ *     else wants them.
+ *  </ul>
+ *
+ * Routerinfos are mostly created by parsing them from a string, in
+ * routerparse.c. We store them to disk on receiving them, and
+ * periodically discard the ones we don't need. On restarting, we
+ * re-read them from disk. (This also applies to extrainfo documents, if
+ * we are configured to fetch them.)
+ *
+ * In order to keep our list of routerinfos up-to-date, we periodically
+ * check whether there are any listed in the latest consensus (or in the
+ * votes from other authorities, if we are an authority) that we don't
+ * have.  (This also applies to extrainfo documents, if we are
+ * configured to fetch them.)
+ *
+ * Almost nothing in Tor should use a routerinfo_t to refer directly to
+ * a relay; instead, almost everything should use node_t (implemented in
+ * nodelist.c), which provides a common interface to routerinfo_t,
+ * routerstatus_t, and microdescriptor_t.
+ *
+ * <br>
+ *
+ * This module also has some of the functions used for choosing random
+ * nodes according to different rules and weights.  Historically, they
+ * were all in this module.  Now, they are spread across this module,
+ * nodelist.c, and networkstatus.c.  (TODO: Fix that.)
+ *
+ * <br>
+ *
+ * (For historical reasons) this module also contains code for handling
+ * the list of fallback directories, the list of directory authorities,
+ * and the list of authority certificates.
+ *
+ * For the directory authorities, we have a list containing the public
+ * identity key, and contact points, for each authority.  The
+ * authorities receive descriptors from relays, and publish consensuses,
+ * descriptors, and microdescriptors.  This list is pre-configured.
+ *
+ * Fallback directories are well-known, stable, but untrusted directory
+ * caches that clients which have not yet bootstrapped can use to get
+ * their first networkstatus consensus, in order to find out where the
+ * Tor network really is.  This list is pre-configured in
+ * fallback_dirs.inc.  Every authority also serves as a fallback.
+ *
+ * Both fallback directories and directory authorities are are
+ * represented by a dir_server_t.
+ *
+ * Authority certificates are signed with authority identity keys; they
+ * are used to authenticate shorter-term authority signing keys. We
+ * fetch them when we find a consensus or a vote that has been signed
+ * with a signing key we don't recognize.  We cache them on disk and
+ * load them on startup.  Authority operators generate them with the
+ * "tor-gencert" utility.
+ *
+ * TODO: Authority certificates should be a separate module.
+ *
+ * TODO: dir_server_t stuff should be in a separate module.
+ **/
+
+#define ROUTERLIST_PRIVATE
+#include "or/or.h"
+#include "lib/err/backtrace.h"
+#include "or/bridges.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "or/circuitstats.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "or/fp_pair.h"
+#include "or/geoip.h"
+#include "or/hibernate.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/reasons.h"
+#include "or/rendcommon.h"
+#include "or/rendservice.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "lib/sandbox/sandbox.h"
+#include "or/torcert.h"
+#include "lib/math/fp.h"
+
+#include "or/dirauth/dirvote.h"
+#include "or/dirauth/mode.h"
+
+#include "or/authority_cert_st.h"
+#include "or/dir_connection_st.h"
+#include "or/dir_server_st.h"
+#include "or/document_signature_st.h"
+#include "or/extrainfo_st.h"
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/vote_routerstatus_st.h"
+
+#include "lib/crypt_ops/digestset.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+// #define DEBUG_ROUTERLIST
+
+/****************************************************************************/
+
+/* Typed wrappers for different digestmap types; used to avoid type
+ * confusion. */
+
+DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t)
+DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
+#define SDMAP_FOREACH(map, keyvar, valvar)                              \
+  DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \
+                    valvar)
+#define RIMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(rimap_to_digestmap(map), keyvar, routerinfo_t *, valvar)
+#define EIMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(eimap_to_digestmap(map), keyvar, extrainfo_t *, valvar)
+#define DSMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
+                    valvar)
+#define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn))
+#define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn))
+#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
+#define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn))
+
+/* Forward declaration for cert_list_t */
+typedef struct cert_list_t cert_list_t;
+
+/* static function prototypes */
+static int compute_weighted_bandwidths(const smartlist_t *sl,
+                                       bandwidth_weight_rule_t rule,
+                                       double **bandwidths_out,
+                                       double *total_bandwidth_out);
+static const routerstatus_t *router_pick_trusteddirserver_impl(
+                const smartlist_t *sourcelist, dirinfo_type_t auth,
+                int flags, int *n_busy_out);
+static const routerstatus_t *router_pick_dirserver_generic(
+                              smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags);
+static void mark_all_dirservers_up(smartlist_t *server_list);
+static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
+static const char *signed_descriptor_get_body_impl(
+                                              const signed_descriptor_t *desc,
+                                              int with_annotations);
+static void list_pending_downloads(digestmap_t *result,
+                                   digest256map_t *result256,
+                                   int purpose, const char *prefix);
+static void list_pending_fpsk_downloads(fp_pair_map_t *result);
+static void launch_dummy_descriptor_download_as_needed(time_t now,
+                                   const or_options_t *options);
+static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
+                                              const char *digest);
+static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                                const char *digest,
+                                                time_t now);
+
+/****************************************************************************/
+
+/** Global list of a dir_server_t object for each directory
+ * authority. */
+static smartlist_t *trusted_dir_servers = NULL;
+/** Global list of dir_server_t objects for all directory authorities
+ * and all fallback directory servers. */
+static smartlist_t *fallback_dir_servers = NULL;
+
+/** List of certificates for a single authority, and download status for
+ * latest certificate.
+ */
+struct cert_list_t {
+  /*
+   * The keys of download status map are cert->signing_key_digest for pending
+   * downloads by (identity digest/signing key digest) pair; functions such
+   * as authority_cert_get_by_digest() already assume these are unique.
+   */
+  struct digest_ds_map_t *dl_status_map;
+  /* There is also a dlstatus for the download by identity key only */
+  download_status_t dl_status_by_id;
+  smartlist_t *certs;
+};
+/** Map from v3 identity key digest to cert_list_t. */
+static digestmap_t *trusted_dir_certs = NULL;
+/** True iff any key certificate in at least one member of
+ * <b>trusted_dir_certs</b> has changed since we last flushed the
+ * certificates to disk. */
+static int trusted_dir_servers_certs_changed = 0;
+
+/** Global list of all of the routers that we know about. */
+static routerlist_t *routerlist = NULL;
+
+/** List of strings for nicknames we've already warned about and that are
+ * still unknown / unavailable. */
+static smartlist_t *warned_nicknames = NULL;
+
+/** The last time we tried to download any routerdesc, or 0 for "never".  We
+ * use this to rate-limit download attempts when the number of routerdescs to
+ * download is low. */
+static time_t last_descriptor_download_attempted = 0;
+
+/** Return the number of directory authorities whose type matches some bit set
+ * in <b>type</b>  */
+int
+get_n_authorities(dirinfo_type_t type)
+{
+  int n = 0;
+  if (!trusted_dir_servers)
+    return 0;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+                    if (ds->type & type)
+                      ++n);
+  return n;
+}
+
+/** Initialise schedule, want_authority, and increment_on in the download
+ * status dlstatus, then call download_status_reset() on it.
+ * It is safe to call this function or download_status_reset() multiple times
+ * on a new dlstatus. But it should *not* be called after a dlstatus has been
+ * used to count download attempts or failures. */
+static void
+download_status_cert_init(download_status_t *dlstatus)
+{
+  dlstatus->schedule = DL_SCHED_CONSENSUS;
+  dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
+  dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
+  dlstatus->last_backoff_position = 0;
+  dlstatus->last_delay_used = 0;
+
+  /* Use the new schedule to set next_attempt_at */
+  download_status_reset(dlstatus);
+}
+
+/** Reset the download status of a specified element in a dsmap */
+static void
+download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
+{
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (!dlstatus) {
+    /* Insert before we reset */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    download_status_cert_init(dlstatus);
+  }
+  tor_assert(dlstatus);
+  /* Go ahead and reset it */
+  download_status_reset(dlstatus);
+}
+
+/**
+ * Return true if the download for this signing key digest in cl is ready
+ * to be re-attempted.
+ */
+static int
+download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                     const char *digest,
+                                     time_t now)
+{
+  int rv = 0;
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (dlstatus) {
+    /* Use download_status_is_ready() */
+    rv = download_status_is_ready(dlstatus, now);
+  } else {
+    /*
+     * If we don't know anything about it, return 1, since we haven't
+     * tried this one before.  We need to create a new entry here,
+     * too.
+     */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    download_status_cert_init(dlstatus);
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    rv = 1;
+  }
+
+  return rv;
+}
+
+/** Helper: Return the cert_list_t for an authority whose authority ID is
+ * <b>id_digest</b>, allocating a new list if necessary. */
+static cert_list_t *
+get_cert_list(const char *id_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs)
+    trusted_dir_certs = digestmap_new();
+  cl = digestmap_get(trusted_dir_certs, id_digest);
+  if (!cl) {
+    cl = tor_malloc_zero(sizeof(cert_list_t));
+    download_status_cert_init(&cl->dl_status_by_id);
+    cl->certs = smartlist_new();
+    cl->dl_status_map = dsmap_new();
+    digestmap_set(trusted_dir_certs, id_digest, cl);
+  }
+  return cl;
+}
+
+/** Return a list of authority ID digests with potentially enumerable lists
+ * of download_status_t objects; used by controller GETINFO queries.
+ */
+
+MOCK_IMPL(smartlist_t *,
+list_authority_ids_with_downloads, (void))
+{
+  smartlist_t *ids = smartlist_new();
+  digestmap_iter_t *i;
+  const char *digest;
+  char *tmp;
+  void *cl;
+
+  if (trusted_dir_certs) {
+    for (i = digestmap_iter_init(trusted_dir_certs);
+         !(digestmap_iter_done(i));
+         i = digestmap_iter_next(trusted_dir_certs, i)) {
+      /*
+       * We always have at least dl_status_by_id to query, so no need to
+       * probe deeper than the existence of a cert_list_t.
+       */
+      digestmap_iter_get(i, &digest, &cl);
+      tmp = tor_malloc(DIGEST_LEN);
+      memcpy(tmp, digest, DIGEST_LEN);
+      smartlist_add(ids, tmp);
+    }
+  }
+  /* else definitely no downloads going since nothing even has a cert list */
+
+  return ids;
+}
+
+/** Given an authority ID digest, return a pointer to the default download
+ * status, or NULL if there is no such entry in trusted_dir_certs */
+
+MOCK_IMPL(download_status_t *,
+id_only_download_status_for_authority_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      dl = &(cl->dl_status_by_id);
+    }
+  }
+
+  return dl;
+}
+
+/** Given an authority ID digest, return a smartlist of signing key digests
+ * for which download_status_t is potentially queryable, or NULL if no such
+ * authority ID digest is known. */
+
+MOCK_IMPL(smartlist_t *,
+list_sk_digests_for_authority_id, (const char *digest))
+{
+  smartlist_t *sks = NULL;
+  cert_list_t *cl;
+  dsmap_iter_t *i;
+  const char *sk_digest;
+  char *tmp;
+  download_status_t *dl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      sks = smartlist_new();
+      if (cl->dl_status_map) {
+        for (i = dsmap_iter_init(cl->dl_status_map);
+             !(dsmap_iter_done(i));
+             i = dsmap_iter_next(cl->dl_status_map, i)) {
+          /* Pull the digest out and add it to the list */
+          dsmap_iter_get(i, &sk_digest, &dl);
+          tmp = tor_malloc(DIGEST_LEN);
+          memcpy(tmp, sk_digest, DIGEST_LEN);
+          smartlist_add(sks, tmp);
+        }
+      }
+    }
+  }
+
+  return sks;
+}
+
+/** Given an authority ID digest and a signing key digest, return the
+ * download_status_t or NULL if none exists. */
+
+MOCK_IMPL(download_status_t *,
+download_status_for_authority_id_and_sk,(const char *id_digest,
+                                         const char *sk_digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl = NULL;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, id_digest);
+    if (cl && cl->dl_status_map) {
+      dl = dsmap_get(cl->dl_status_map, sk_digest);
+    }
+  }
+
+  return dl;
+}
+
+#define cert_list_free(val) \
+  FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
+
+/** Release all space held by a cert_list_t */
+static void
+cert_list_free_(cert_list_t *cl)
+{
+  if (!cl)
+    return;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+                    authority_cert_free(cert));
+  smartlist_free(cl->certs);
+  dsmap_free(cl->dl_status_map, tor_free_);
+  tor_free(cl);
+}
+
+/** Wrapper for cert_list_free so we can pass it to digestmap_free */
+static void
+cert_list_free_void(void *cl)
+{
+  cert_list_free_(cl);
+}
+
+/** Reload the cached v3 key certificates from the cached-certs file in
+ * the data directory. Return 0 on success, -1 on failure. */
+int
+trusted_dirs_reload_certs(void)
+{
+  char *filename;
+  char *contents;
+  int r;
+
+  filename = get_cachedir_fname("cached-certs");
+  contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  if (!contents)
+    return 0;
+  r = trusted_dirs_load_certs_from_string(
+        contents,
+        TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
+  tor_free(contents);
+  return r;
+}
+
+/** Helper: return true iff we already have loaded the exact cert
+ * <b>cert</b>. */
+static inline int
+already_have_cert(authority_cert_t *cert)
+{
+  cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+  {
+    if (tor_memeq(c->cache_info.signed_descriptor_digest,
+                cert->cache_info.signed_descriptor_digest,
+                DIGEST_LEN))
+      return 1;
+  });
+  return 0;
+}
+
+/** Load a bunch of new key certificates from the string <b>contents</b>.  If
+ * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
+ * from the cache, and we don't need to flush them to disk.  If we are a
+ * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
+ * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
+ * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST.  If <b>flush</b> is true, we
+ * need to flush any changed certificates to disk now.  Return 0 on success,
+ * -1 if any certs fail to parse.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved certificates from, so try it first to
+ * fetch any missing certificates.
+ */
+int
+trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                    int flush, const char *source_dir)
+{
+  dir_server_t *ds;
+  const char *s, *eos;
+  int failure_code = 0;
+  int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
+  int added_trusted_cert = 0;
+
+  for (s = contents; *s; s = eos) {
+    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    cert_list_t *cl;
+    if (!cert) {
+      failure_code = -1;
+      break;
+    }
+    ds = trusteddirserver_get_by_v3_auth_digest(
+                                       cert->cache_info.identity_digest);
+    log_debug(LD_DIR, "Parsed certificate for %s",
+              ds ? ds->nickname : "unknown authority");
+
+    if (already_have_cert(cert)) {
+      /* we already have this one. continue. */
+      log_info(LD_DIR, "Skipping %s certificate for %s that we "
+               "already have.",
+               from_store ? "cached" : "downloaded",
+               ds ? ds->nickname : "an old or new authority");
+
+      /*
+       * A duplicate on download should be treated as a failure, so we call
+       * authority_cert_dl_failed() to reset the download status to make sure
+       * we can't try again.  Since we've implemented the fp-sk mechanism
+       * to download certs by signing key, this should be much rarer than it
+       * was and is perhaps cause for concern.
+       */
+      if (!from_store) {
+        if (authdir_mode(get_options())) {
+          log_warn(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        } else {
+          log_info(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        }
+
+        /*
+         * This is where we care about the source; authority_cert_dl_failed()
+         * needs to know whether the download was by fp or (fp,sk) pair to
+         * twiddle the right bit in the download map.
+         */
+        if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   NULL, 404);
+        } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   cert->signing_key_digest, 404);
+        }
+      }
+
+      authority_cert_free(cert);
+      continue;
+    }
+
+    if (ds) {
+      added_trusted_cert = 1;
+      log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
+               "signing key %s", from_store ? "cached" : "downloaded",
+               ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
+    } else {
+      int adding = we_want_to_fetch_unknown_auth_certs(get_options());
+      log_info(LD_DIR, "%s %s certificate for unrecognized directory "
+               "authority with signing key %s",
+               adding ? "Adding" : "Not adding",
+               from_store ? "cached" : "downloaded",
+               hex_str(cert->signing_key_digest,DIGEST_LEN));
+      if (!adding) {
+        authority_cert_free(cert);
+        continue;
+      }
+    }
+
+    cl = get_cert_list(cert->cache_info.identity_digest);
+    smartlist_add(cl->certs, cert);
+    if (ds && cert->cache_info.published_on > ds->addr_current_at) {
+      /* Check to see whether we should update our view of the authority's
+       * address. */
+      if (cert->addr && cert->dir_port &&
+          (ds->addr != cert->addr ||
+           ds->dir_port != cert->dir_port)) {
+        char *a = tor_dup_ip(cert->addr);
+        log_notice(LD_DIR, "Updating address for directory authority %s "
+                   "from %s:%d to %s:%d based on certificate.",
+                   ds->nickname, ds->address, (int)ds->dir_port,
+                   a, cert->dir_port);
+        tor_free(a);
+        ds->addr = cert->addr;
+        ds->dir_port = cert->dir_port;
+      }
+      ds->addr_current_at = cert->cache_info.published_on;
+    }
+
+    if (!from_store)
+      trusted_dir_servers_certs_changed = 1;
+  }
+
+  if (flush)
+    trusted_dirs_flush_certs_to_disk();
+
+  /* call this even if failure_code is <0, since some certs might have
+   * succeeded, but only pass source_dir if there were no failures,
+   * and at least one more authority certificate was added to the store.
+   * This avoids retrying a directory that's serving bad or entirely duplicate
+   * certificates. */
+  if (failure_code == 0 && added_trusted_cert) {
+    networkstatus_note_certs_arrived(source_dir);
+  } else {
+    networkstatus_note_certs_arrived(NULL);
+  }
+
+  return failure_code;
+}
+
+/** Save all v3 key certificates to the cached-certs file. */
+void
+trusted_dirs_flush_certs_to_disk(void)
+{
+  char *filename;
+  smartlist_t *chunks;
+
+  if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
+    return;
+
+  chunks = smartlist_new();
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+          {
+            sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
+            c->bytes = cert->cache_info.signed_descriptor_body;
+            c->len = cert->cache_info.signed_descriptor_len;
+            smartlist_add(chunks, c);
+          });
+  } DIGESTMAP_FOREACH_END;
+
+  filename = get_cachedir_fname("cached-certs");
+  if (write_chunks_to_file(filename, chunks, 0, 0)) {
+    log_warn(LD_FS, "Error writing certificates to disk.");
+  }
+  tor_free(filename);
+  SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
+  smartlist_free(chunks);
+
+  trusted_dir_servers_certs_changed = 0;
+}
+
+static int
+compare_certs_by_pubdates(const void **_a, const void **_b)
+{
+  const authority_cert_t *cert1 = *_a, *cert2=*_b;
+
+  if (cert1->cache_info.published_on < cert2->cache_info.published_on)
+    return -1;
+  else if (cert1->cache_info.published_on >  cert2->cache_info.published_on)
+    return 1;
+  else
+    return 0;
+}
+
+/** Remove all expired v3 authority certificates that have been superseded for
+ * more than 48 hours or, if not expired, that were published more than 7 days
+ * before being superseded. (If the most recent cert was published more than 48
+ * hours ago, then we aren't going to get any consensuses signed with older
+ * keys.) */
+static void
+trusted_dirs_remove_old_certs(void)
+{
+  time_t now = time(NULL);
+#define DEAD_CERT_LIFETIME (2*24*60*60)
+#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    /* Sort the list from first-published to last-published */
+    smartlist_sort(cl->certs, compare_certs_by_pubdates);
+
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
+        /* This is the most recently published cert.  Keep it. */
+        continue;
+      }
+      authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
+      const time_t next_cert_published = next_cert->cache_info.published_on;
+      if (next_cert_published > now) {
+        /* All later certs are published in the future. Keep everything
+         * we didn't discard. */
+        break;
+      }
+      int should_remove = 0;
+      if (cert->expires + DEAD_CERT_LIFETIME < now) {
+        /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
+         * Remove it. */
+        should_remove = 1;
+      } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
+        /* Certificate has been superseded for OLD_CERT_LIFETIME.
+         * Remove it.
+         */
+        should_remove = 1;
+      }
+      if (should_remove) {
+        SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
+        authority_cert_free(cert);
+        trusted_dir_servers_certs_changed = 1;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+
+  } DIGESTMAP_FOREACH_END;
+#undef DEAD_CERT_LIFETIME
+#undef OLD_CERT_LIFETIME
+
+  trusted_dirs_flush_certs_to_disk();
+}
+
+/** Return the newest v3 authority certificate whose v3 authority identity key
+ * has digest <b>id_digest</b>.  Return NULL if no such authority is known,
+ * or it has no certificate. */
+authority_cert_t *
+authority_cert_get_newest_by_id(const char *id_digest)
+{
+  cert_list_t *cl;
+  authority_cert_t *best = NULL;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+  {
+    if (!best || cert->cache_info.published_on > best->cache_info.published_on)
+      best = cert;
+  });
+  return best;
+}
+
+/** Return the newest v3 authority certificate whose directory signing key has
+ * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
+ */
+authority_cert_t *
+authority_cert_get_by_sk_digest(const char *sk_digest)
+{
+  authority_cert_t *c;
+  if (!trusted_dir_certs)
+    return NULL;
+
+  if ((c = get_my_v3_authority_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+  if ((c = get_my_v3_legacy_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    {
+      if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+        return cert;
+    });
+  } DIGESTMAP_FOREACH_END;
+  return NULL;
+}
+
+/** Return the v3 authority certificate with signing key matching
+ * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
+ * Return NULL if no such authority is known. */
+authority_cert_t *
+authority_cert_get_by_digests(const char *id_digest,
+                              const char *sk_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+      return cert; );
+
+  return NULL;
+}
+
+/** Add every known authority_cert_t to <b>certs_out</b>. */
+void
+authority_cert_get_all(smartlist_t *certs_out)
+{
+  tor_assert(certs_out);
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+                      smartlist_add(certs_out, c));
+  } DIGESTMAP_FOREACH_END;
+}
+
+/** Called when an attempt to download a certificate with the authority with
+ * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
+ * fails with HTTP response code <b>status</b>: remember the failure, so we
+ * don't try again immediately. */
+void
+authority_cert_dl_failed(const char *id_digest,
+                         const char *signing_key_digest, int status)
+{
+  cert_list_t *cl;
+  download_status_t *dlstatus = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return;
+
+  /*
+   * Are we noting a failed download of the latest cert for the id digest,
+   * or of a download by (id, signing key) digest pair?
+   */
+  if (!signing_key_digest) {
+    /* Just by id digest */
+    download_status_failed(&cl->dl_status_by_id, status);
+  } else {
+    /* Reset by (id, signing key) digest pair
+     *
+     * Look for a download_status_t in the map with this digest
+     */
+    dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
+    /* Got one? */
+    if (dlstatus) {
+      download_status_failed(dlstatus, status);
+    } else {
+      /*
+       * Do this rather than hex_str(), since hex_str clobbers
+       * old results and we call twice in the param list.
+       */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    id_digest, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    signing_key_digest, DIGEST_LEN);
+      log_warn(LD_BUG,
+               "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
+               "status %d, but knew nothing about the download.",
+               id_digest_str, sk_digest_str, status);
+    }
+  }
+}
+
+static const char *BAD_SIGNING_KEYS[] = {
+  "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
+  "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
+  "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
+  "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
+  "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
+  "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
+  "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
+  "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
+  "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
+  "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
+  NULL,
+};
+
+/** Return true iff <b>cert</b> authenticates some atuhority signing key
+ * which, because of the old openssl heartbleed vulnerability, should
+ * never be trusted. */
+int
+authority_cert_is_blacklisted(const authority_cert_t *cert)
+{
+  char hex_digest[HEX_DIGEST_LEN+1];
+  int i;
+  base16_encode(hex_digest, sizeof(hex_digest),
+                cert->signing_key_digest, sizeof(cert->signing_key_digest));
+
+  for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
+    if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/** Return true iff when we've been getting enough failures when trying to
+ * download the certificate with ID digest <b>id_digest</b> that we're willing
+ * to start bugging the user about it. */
+int
+authority_cert_dl_looks_uncertain(const char *id_digest)
+{
+#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
+  cert_list_t *cl;
+  int n_failures;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return 0;
+
+  n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
+  return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
+}
+
+/* Fetch the authority certificates specified in resource.
+ * If we are a bridge client, and node is a configured bridge, fetch from node
+ * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
+ * rs. Otherwise, fetch from a random directory mirror. */
+static void
+authority_certs_fetch_resource_impl(const char *resource,
+                                    const char *dir_hint,
+                                    const node_t *node,
+                                    const routerstatus_t *rs)
+{
+  const or_options_t *options = get_options();
+  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                                            resource);
+
+  /* Make sure bridge clients never connect to anything but a bridge */
+  if (options->UseBridges) {
+    if (node && !node_is_a_configured_bridge(node)) {
+      /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
+      get_via_tor = 1;
+    } else if (!node) {
+      /* If we're using bridges, and there's no node, use a 3-hop path. */
+      get_via_tor = 1;
+    }
+  }
+
+  const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
+                                                    : DIRIND_ONEHOP;
+
+  directory_request_t *req = NULL;
+  /* If we've just downloaded a consensus from a bridge, re-use that
+   * bridge */
+  if (options->UseBridges && node && node->ri && !get_via_tor) {
+    /* clients always make OR connections to bridges */
+    tor_addr_port_t or_ap;
+    /* we are willing to use a non-preferred address if we need to */
+    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+                                         &or_ap);
+
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_or_addr_port(req, &or_ap);
+    if (dir_hint)
+      directory_request_set_directory_id_digest(req, dir_hint);
+  } else if (rs) {
+    /* And if we've just downloaded a consensus from a directory, re-use that
+     * directory */
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_routerstatus(req, rs);
+  }
+
+  if (req) {
+    /* We've set up a request object -- fill in the other request fields, and
+     * send the request.  */
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
+    return;
+  }
+
+  /* Otherwise, we want certs from a random fallback or directory
+   * mirror, because they will almost always succeed. */
+  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                               resource, PDS_RETRY_IF_NO_SERVERS,
+                               DL_WANT_ANY_DIRSERVER);
+}
+
+/** Try to download any v3 authority certificates that we may be missing.  If
+ * <b>status</b> is provided, try to get all the ones that were used to sign
+ * <b>status</b>.  Additionally, try to have a non-expired certificate for
+ * every V3 authority in trusted_dir_servers.  Don't fetch certificates we
+ * already have.
+ *
+ * If dir_hint is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved a consensus or certificates from, so try
+ * it first to fetch any missing certificates.
+ **/
+void
+authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                              const char *dir_hint)
+{
+  /*
+   * The pending_id digestmap tracks pending certificate downloads by
+   * identity digest; the pending_cert digestmap tracks pending downloads
+   * by (identity digest, signing key digest) pairs.
+   */
+  digestmap_t *pending_id;
+  fp_pair_map_t *pending_cert;
+  /*
+   * The missing_id_digests smartlist will hold a list of id digests
+   * we want to fetch the newest cert for; the missing_cert_digests
+   * smartlist will hold a list of fp_pair_t with an identity and
+   * signing key digest.
+   */
+  smartlist_t *missing_cert_digests, *missing_id_digests;
+  char *resource = NULL;
+  cert_list_t *cl;
+  const or_options_t *options = get_options();
+  const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
+  fp_pair_t *fp_tmp = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+
+  pending_cert = fp_pair_map_new();
+  pending_id = digestmap_new();
+  missing_cert_digests = smartlist_new();
+  missing_id_digests = smartlist_new();
+
+  /*
+   * First, we get the lists of already pending downloads so we don't
+   * duplicate effort.
+   */
+  list_pending_downloads(pending_id, NULL,
+                         DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
+  list_pending_fpsk_downloads(pending_cert);
+
+  /*
+   * Now, we download any trusted authority certs we don't have by
+   * identity digest only.  This gets the latest cert for that authority.
+   */
+  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) {
+    int found = 0;
+    if (!(ds->type & V3_DIRINFO))
+      continue;
+    if (smartlist_contains_digest(missing_id_digests,
+                                  ds->v3_identity_digest))
+      continue;
+    cl = get_cert_list(ds->v3_identity_digest);
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (now < cert->expires) {
+        /* It's not expired, and we weren't looking for something to
+         * verify a consensus with.  Call it done. */
+        download_status_reset(&(cl->dl_status_by_id));
+        /* No sense trying to download it specifically by signing key hash */
+        download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
+        found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+    if (!found &&
+        download_status_is_ready(&(cl->dl_status_by_id), now) &&
+        !digestmap_get(pending_id, ds->v3_identity_digest)) {
+      log_info(LD_DIR,
+               "No current certificate known for authority %s "
+               "(ID digest %s); launching request.",
+               ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
+      smartlist_add(missing_id_digests, ds->v3_identity_digest);
+    }
+  } SMARTLIST_FOREACH_END(ds);
+
+  /*
+   * Next, if we have a consensus, scan through it and look for anything
+   * signed with a key from a cert we don't have.  Those get downloaded
+   * by (fp,sk) pair, but if we don't know any certs at all for the fp
+   * (identity digest), and it's one of the trusted dir server certs
+   * we started off above or a pending download in pending_id, don't
+   * try to get it yet.  Most likely, the one we'll get for that will
+   * have the right signing key too, and we'd just be downloading
+   * redundantly.
+   */
+  if (status) {
+    SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
+                            voter) {
+      if (!smartlist_len(voter->sigs))
+        continue; /* This authority never signed this consensus, so don't
+                   * go looking for a cert with key digest 0000000000. */
+      if (!keep_unknown &&
+          !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
+        continue; /* We don't want unknown certs, and we don't know this
+                   * authority.*/
+
+      /*
+       * If we don't know *any* cert for this authority, and a download by ID
+       * is pending or we added it to missing_id_digests above, skip this
+       * one for now to avoid duplicate downloads.
+       */
+      cl = get_cert_list(voter->identity_digest);
+      if (smartlist_len(cl->certs) == 0) {
+        /* We have no certs at all for this one */
+
+        /* Do we have a download of one pending? */
+        if (digestmap_get(pending_id, voter->identity_digest))
+          continue;
+
+        /*
+         * Are we about to launch a download of one due to the trusted
+         * dir server check above?
+         */
+        if (smartlist_contains_digest(missing_id_digests,
+                                      voter->identity_digest))
+          continue;
+      }
+
+      SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+        authority_cert_t *cert =
+          authority_cert_get_by_digests(voter->identity_digest,
+                                        sig->signing_key_digest);
+        if (cert) {
+          if (now < cert->expires)
+            download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
+          continue;
+        }
+        if (download_status_is_ready_by_sk_in_cl(
+              cl, sig->signing_key_digest, now) &&
+            !fp_pair_map_get_by_digests(pending_cert,
+                                        voter->identity_digest,
+                                        sig->signing_key_digest)) {
+          /*
+           * Do this rather than hex_str(), since hex_str clobbers
+           * old results and we call twice in the param list.
+           */
+          base16_encode(id_digest_str, sizeof(id_digest_str),
+                        voter->identity_digest, DIGEST_LEN);
+          base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                        sig->signing_key_digest, DIGEST_LEN);
+
+          if (voter->nickname) {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority %s "
+                     "(ID digest %s) with signing key %s: "
+                     "launching request.",
+                     voter->nickname, id_digest_str, sk_digest_str);
+          } else {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority ID digest "
+                     "%s with signing key %s: launching request.",
+                     id_digest_str, sk_digest_str);
+          }
+
+          /* Allocate a new fp_pair_t to append */
+          fp_tmp = tor_malloc(sizeof(*fp_tmp));
+          memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
+          memcpy(fp_tmp->second, sig->signing_key_digest,
+                 sizeof(fp_tmp->second));
+          smartlist_add(missing_cert_digests, fp_tmp);
+        }
+      } SMARTLIST_FOREACH_END(sig);
+    } SMARTLIST_FOREACH_END(voter);
+  }
+
+  /* Bridge clients look up the node for the dir_hint */
+  const node_t *node = NULL;
+  /* All clients, including bridge clients, look up the routerstatus for the
+   * dir_hint */
+  const routerstatus_t *rs = NULL;
+
+  /* If we still need certificates, try the directory that just successfully
+   * served us a consensus or certificates.
+   * As soon as the directory fails to provide additional certificates, we try
+   * another, randomly selected directory. This avoids continual retries.
+   * (We only ever have one outstanding request per certificate.)
+   */
+  if (dir_hint) {
+    if (options->UseBridges) {
+      /* Bridge clients try the nodelist. If the dir_hint is from an authority,
+       * or something else fetched over tor, we won't find the node here, but
+       * we will find the rs. */
+      node = node_get_by_id(dir_hint);
+    }
+
+    /* All clients try the consensus routerstatus, then the fallback
+     * routerstatus */
+    rs = router_get_consensus_status_by_id(dir_hint);
+    if (!rs) {
+      /* This will also find authorities */
+      const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
+                                                                    dir_hint);
+      if (ds) {
+        rs = &ds->fake_status;
+      }
+    }
+
+    if (!node && !rs) {
+      log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
+               "no routerstatus could be found for it.",
+               options->UseBridges ? "no node and " : "",
+               hex_str(dir_hint, DIGEST_LEN));
+    }
+  }
+
+  /* Do downloads by identity digest */
+  if (smartlist_len(missing_id_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fps = smartlist_new();
+
+    smartlist_add_strdup(fps, "fp/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
+      char *fp = NULL;
+
+      if (digestmap_get(pending_id, d))
+        continue;
+
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d, DIGEST_LEN);
+
+      if (need_plus) {
+        tor_asprintf(&fp, "+%s", id_digest_str);
+      } else {
+        /* No need for tor_asprintf() in this case; first one gets no '+' */
+        fp = tor_strdup(id_digest_str);
+        need_plus = 1;
+      }
+
+      smartlist_add(fps, fp);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fps) > 1) {
+      resource = smartlist_join_strings(fps, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else we didn't add any: they were all pending */
+
+    SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
+    smartlist_free(fps);
+  }
+
+  /* Do downloads by identity digest/signing key pair */
+  if (smartlist_len(missing_cert_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fp_pairs = smartlist_new();
+
+    smartlist_add_strdup(fp_pairs, "fp-sk/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
+      char *fp_pair = NULL;
+
+      if (fp_pair_map_get(pending_cert, d))
+        continue;
+
+      /* Construct string encodings of the digests */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d->first, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    d->second, DIGEST_LEN);
+
+      /* Now tor_asprintf() */
+      if (need_plus) {
+        tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
+      } else {
+        /* First one in the list doesn't get a '+' */
+        tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
+        need_plus = 1;
+      }
+
+      /* Add it to the list of pairs to request */
+      smartlist_add(fp_pairs, fp_pair);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fp_pairs) > 1) {
+      resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else they were all pending */
+
+    SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
+    smartlist_free(fp_pairs);
+  }
+
+  smartlist_free(missing_id_digests);
+  SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
+  smartlist_free(missing_cert_digests);
+  digestmap_free(pending_id, NULL);
+  fp_pair_map_free(pending_cert, NULL);
+}
+
+/* Router descriptor storage.
+ *
+ * Routerdescs are stored in a big file, named "cached-descriptors".  As new
+ * routerdescs arrive, we append them to a journal file named
+ * "cached-descriptors.new".
+ *
+ * From time to time, we replace "cached-descriptors" with a new file
+ * containing only the live, non-superseded descriptors, and clear
+ * cached-routers.new.
+ *
+ * On startup, we read both files.
+ */
+
+/** Helper: return 1 iff the router log is so big we want to rebuild the
+ * store. */
+static int
+router_should_rebuild_store(desc_store_t *store)
+{
+  if (store->store_len > (1<<16))
+    return (store->journal_len > store->store_len / 2 ||
+            store->bytes_dropped > store->store_len / 2);
+  else
+    return store->journal_len > (1<<15);
+}
+
+/** Return the desc_store_t in <b>rl</b> that should be used to store
+ * <b>sd</b>. */
+static inline desc_store_t *
+desc_get_store(routerlist_t *rl, const signed_descriptor_t *sd)
+{
+  if (sd->is_extrainfo)
+    return &rl->extrainfo_store;
+  else
+    return &rl->desc_store;
+}
+
+/** Add the signed_descriptor_t in <b>desc</b> to the router
+ * journal; change its saved_location to SAVED_IN_JOURNAL and set its
+ * offset appropriately. */
+static int
+signed_desc_append_to_journal(signed_descriptor_t *desc,
+                              desc_store_t *store)
+{
+  char *fname = get_cachedir_fname_suffix(store->fname_base, ".new");
+  const char *body = signed_descriptor_get_body_impl(desc,1);
+  size_t len = desc->signed_descriptor_len + desc->annotations_len;
+
+  if (append_bytes_to_file(fname, body, len, 1)) {
+    log_warn(LD_FS, "Unable to store router descriptor");
+    tor_free(fname);
+    return -1;
+  }
+  desc->saved_location = SAVED_IN_JOURNAL;
+  tor_free(fname);
+
+  desc->saved_offset = store->journal_len;
+  store->journal_len += len;
+
+  return 0;
+}
+
+/** Sorting helper: return &lt;0, 0, or &gt;0 depending on whether the
+ * signed_descriptor_t* in *<b>a</b> is older, the same age as, or newer than
+ * the signed_descriptor_t* in *<b>b</b>. */
+static int
+compare_signed_descriptors_by_age_(const void **_a, const void **_b)
+{
+  const signed_descriptor_t *r1 = *_a, *r2 = *_b;
+  return (int)(r1->published_on - r2->published_on);
+}
+
+#define RRS_FORCE 1
+#define RRS_DONT_REMOVE_OLD 2
+
+/** If the journal of <b>store</b> is too long, or if RRS_FORCE is set in
+ * <b>flags</b>, then atomically replace the saved router store with the
+ * routers currently in our routerlist, and clear the journal.  Unless
+ * RRS_DONT_REMOVE_OLD is set in <b>flags</b>, delete expired routers before
+ * rebuilding the store.  Return 0 on success, -1 on failure.
+ */
+static int
+router_rebuild_store(int flags, desc_store_t *store)
+{
+  smartlist_t *chunk_list = NULL;
+  char *fname = NULL, *fname_tmp = NULL;
+  int r = -1;
+  off_t offset = 0;
+  smartlist_t *signed_descriptors = NULL;
+  int nocache=0;
+  size_t total_expected_len = 0;
+  int had_any;
+  int force = flags & RRS_FORCE;
+
+  if (!force && !router_should_rebuild_store(store)) {
+    r = 0;
+    goto done;
+  }
+  if (!routerlist) {
+    r = 0;
+    goto done;
+  }
+
+  if (store->type == EXTRAINFO_STORE)
+    had_any = !eimap_isempty(routerlist->extra_info_map);
+  else
+    had_any = (smartlist_len(routerlist->routers)+
+               smartlist_len(routerlist->old_routers))>0;
+
+  /* Don't save deadweight. */
+  if (!(flags & RRS_DONT_REMOVE_OLD))
+    routerlist_remove_old_routers();
+
+  log_info(LD_DIR, "Rebuilding %s cache", store->description);
+
+  fname = get_cachedir_fname(store->fname_base);
+  fname_tmp = get_cachedir_fname_suffix(store->fname_base, ".tmp");
+
+  chunk_list = smartlist_new();
+
+  /* We sort the routers by age to enhance locality on disk. */
+  signed_descriptors = smartlist_new();
+  if (store->type == EXTRAINFO_STORE) {
+    eimap_iter_t *iter;
+    for (iter = eimap_iter_init(routerlist->extra_info_map);
+         !eimap_iter_done(iter);
+         iter = eimap_iter_next(routerlist->extra_info_map, iter)) {
+      const char *key;
+      extrainfo_t *ei;
+      eimap_iter_get(iter, &key, &ei);
+      smartlist_add(signed_descriptors, &ei->cache_info);
+    }
+  } else {
+    SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd,
+                      smartlist_add(signed_descriptors, sd));
+    SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
+                      smartlist_add(signed_descriptors, &ri->cache_info));
+  }
+
+  smartlist_sort(signed_descriptors, compare_signed_descriptors_by_age_);
+
+  /* Now, add the appropriate members to chunk_list */
+  SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) {
+      sized_chunk_t *c;
+      const char *body = signed_descriptor_get_body_impl(sd, 1);
+      if (!body) {
+        log_warn(LD_BUG, "No descriptor available for router.");
+        goto done;
+      }
+      if (sd->do_not_cache) {
+        ++nocache;
+        continue;
+      }
+      c = tor_malloc(sizeof(sized_chunk_t));
+      c->bytes = body;
+      c->len = sd->signed_descriptor_len + sd->annotations_len;
+      total_expected_len += c->len;
+      smartlist_add(chunk_list, c);
+  } SMARTLIST_FOREACH_END(sd);
+
+  if (write_chunks_to_file(fname_tmp, chunk_list, 1, 1)<0) {
+    log_warn(LD_FS, "Error writing router store to disk.");
+    goto done;
+  }
+
+  /* Our mmap is now invalid. */
+  if (store->mmap) {
+    int res = tor_munmap_file(store->mmap);
+    store->mmap = NULL;
+    if (res != 0) {
+      log_warn(LD_FS, "Unable to munmap route store in %s", fname);
+    }
+  }
+
+  if (replace_file(fname_tmp, fname)<0) {
+    log_warn(LD_FS, "Error replacing old router store: %s", strerror(errno));
+    goto done;
+  }
+
+  errno = 0;
+  store->mmap = tor_mmap_file(fname);
+  if (! store->mmap) {
+    if (errno == ERANGE) {
+      /* empty store.*/
+      if (total_expected_len) {
+        log_warn(LD_FS, "We wrote some bytes to a new descriptor file at '%s',"
+                 " but when we went to mmap it, it was empty!", fname);
+      } else if (had_any) {
+        log_info(LD_FS, "We just removed every descriptor in '%s'.  This is "
+                 "okay if we're just starting up after a long time. "
+                 "Otherwise, it's a bug.", fname);
+      }
+    } else {
+      log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname);
+    }
+  }
+
+  log_info(LD_DIR, "Reconstructing pointers into cache");
+
+  offset = 0;
+  SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) {
+      if (sd->do_not_cache)
+        continue;
+      sd->saved_location = SAVED_IN_CACHE;
+      if (store->mmap) {
+        tor_free(sd->signed_descriptor_body); // sets it to null
+        sd->saved_offset = offset;
+      }
+      offset += sd->signed_descriptor_len + sd->annotations_len;
+      signed_descriptor_get_body(sd); /* reconstruct and assert */
+  } SMARTLIST_FOREACH_END(sd);
+
+  tor_free(fname);
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
+  write_str_to_file(fname, "", 1);
+
+  r = 0;
+  store->store_len = (size_t) offset;
+  store->journal_len = 0;
+  store->bytes_dropped = 0;
+ done:
+  smartlist_free(signed_descriptors);
+  tor_free(fname);
+  tor_free(fname_tmp);
+  if (chunk_list) {
+    SMARTLIST_FOREACH(chunk_list, sized_chunk_t *, c, tor_free(c));
+    smartlist_free(chunk_list);
+  }
+
+  return r;
+}
+
+/** Helper: Reload a cache file and its associated journal, setting metadata
+ * appropriately.  If <b>extrainfo</b> is true, reload the extrainfo store;
+ * else reload the router descriptor store. */
+static int
+router_reload_router_list_impl(desc_store_t *store)
+{
+  char *fname = NULL, *contents = NULL;
+  struct stat st;
+  int extrainfo = (store->type == EXTRAINFO_STORE);
+  store->journal_len = store->store_len = 0;
+
+  fname = get_cachedir_fname(store->fname_base);
+
+  if (store->mmap) {
+    /* get rid of it first */
+    int res = tor_munmap_file(store->mmap);
+    store->mmap = NULL;
+    if (res != 0) {
+      log_warn(LD_FS, "Failed to munmap %s", fname);
+      tor_free(fname);
+      return -1;
+    }
+  }
+
+  store->mmap = tor_mmap_file(fname);
+  if (store->mmap) {
+    store->store_len = store->mmap->size;
+    if (extrainfo)
+      router_load_extrainfo_from_string(store->mmap->data,
+                                        store->mmap->data+store->mmap->size,
+                                        SAVED_IN_CACHE, NULL, 0);
+    else
+      router_load_routers_from_string(store->mmap->data,
+                                      store->mmap->data+store->mmap->size,
+                                      SAVED_IN_CACHE, NULL, 0, NULL);
+  }
+
+  tor_free(fname);
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
+  /* don't load empty files - we wouldn't get any data, even if we tried */
+  if (file_status(fname) == FN_FILE)
+    contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
+  if (contents) {
+    if (extrainfo)
+      router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL,
+                                        NULL, 0);
+    else
+      router_load_routers_from_string(contents, NULL, SAVED_IN_JOURNAL,
+                                      NULL, 0, NULL);
+    store->journal_len = (size_t) st.st_size;
+    tor_free(contents);
+  }
+
+  tor_free(fname);
+
+  if (store->journal_len) {
+    /* Always clear the journal on startup.*/
+    router_rebuild_store(RRS_FORCE, store);
+  } else if (!extrainfo) {
+    /* Don't cache expired routers. (This is in an else because
+     * router_rebuild_store() also calls remove_old_routers().) */
+    routerlist_remove_old_routers();
+  }
+
+  return 0;
+}
+
+/** Load all cached router descriptors and extra-info documents from the
+ * store. Return 0 on success and -1 on failure.
+ */
+int
+router_reload_router_list(void)
+{
+  routerlist_t *rl = router_get_routerlist();
+  if (router_reload_router_list_impl(&rl->desc_store))
+    return -1;
+  if (router_reload_router_list_impl(&rl->extrainfo_store))
+    return -1;
+  return 0;
+}
+
+/** Return a smartlist containing a list of dir_server_t * for all
+ * known trusted dirservers.  Callers must not modify the list or its
+ * contents.
+ */
+const smartlist_t *
+router_get_trusted_dir_servers(void)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+
+  return trusted_dir_servers;
+}
+
+const smartlist_t *
+router_get_fallback_dir_servers(void)
+{
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  return fallback_dir_servers;
+}
+
+/** Try to find a running dirserver that supports operations of <b>type</b>.
+ *
+ * If there are no running dirservers in our routerlist and the
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
+ * (including authorities) as running again, and pick one.
+ *
+ * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
+ * dirservers that we can't reach.
+ *
+ * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
+ * (if we're a dirserver).
+ *
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
+ */
+const routerstatus_t *
+router_pick_directory_server(dirinfo_type_t type, int flags)
+{
+  int busy = 0;
+  const routerstatus_t *choice;
+
+  if (!routerlist)
+    return NULL;
+
+  choice = router_pick_directory_server_impl(type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No reachable router entries for dirservers. "
+           "Trying them all again.");
+  /* mark all fallback directory mirrors as up again */
+  mark_all_dirservers_up(fallback_dir_servers);
+  /* try again */
+  choice = router_pick_directory_server_impl(type, flags, NULL);
+  return choice;
+}
+
+/** Return the dir_server_t for the directory authority whose identity
+ * key hashes to <b>digest</b>, or NULL if no such authority is known.
+ */
+dir_server_t *
+router_get_trusteddirserver_by_digest(const char *digest)
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return the dir_server_t for the fallback dirserver whose identity
+ * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of
+ * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir
+ * and UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+dir_server_t *
+router_get_fallback_dirserver_by_digest(const char *digest)
+{
+  if (!fallback_dir_servers)
+    return NULL;
+
+  if (!digest)
+    return NULL;
+
+  SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>,
+ * or 0 if no such fallback is in the list of fallback_dir_servers.
+ * (fallback_dir_servers is affected by the FallbackDir and
+ * UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+int
+router_digest_is_fallback_dir(const char *digest)
+{
+  return (router_get_fallback_dirserver_by_digest(digest) != NULL);
+}
+
+/** Return the dir_server_t for the directory authority whose
+ * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
+ * is known.
+ */
+MOCK_IMPL(dir_server_t *,
+trusteddirserver_get_by_v3_auth_digest, (const char *digest))
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
+           (ds->type & V3_DIRINFO))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Try to find a running directory authority. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_trusteddirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(trusted_dir_servers, type, flags);
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(fallback_dir_servers, type, flags);
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+static const routerstatus_t *
+router_pick_dirserver_generic(smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags)
+{
+  const routerstatus_t *choice;
+  int busy = 0;
+
+  choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No dirservers are reachable. Trying them all again.");
+  mark_all_dirservers_up(sourcelist);
+  return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
+}
+
+/* Check if we already have a directory fetch from ap, for serverdesc
+ * (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0.
+ */
+STATIC int
+router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
+                               int microdesc)
+{
+  if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) {
+    return 0;
+  }
+
+  /* XX/teor - we're not checking tunnel connections here, see #17848
+   */
+  if (serverdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC)
+  || connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) {
+    return 1;
+  }
+
+  if (microdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* Check if we already have a directory fetch from the ipv4 or ipv6
+ * router, for serverdesc (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ */
+static int
+router_is_already_dir_fetching_(uint32_t ipv4_addr,
+                                const tor_addr_t *ipv6_addr,
+                                uint16_t dir_port,
+                                int serverdesc,
+                                int microdesc)
+{
+  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
+
+  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
+  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
+  ipv4_dir_ap.port = dir_port;
+  tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
+  ipv6_dir_ap.port = dir_port;
+
+  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
+       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
+}
+
+#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0
+#endif
+
+/* Log a message if rs is not found or not a preferred address */
+static void
+router_picked_poor_directory_log(const routerstatus_t *rs)
+{
+  const networkstatus_t *usable_consensus;
+  usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL),
+                                                 usable_consensus_flavor());
+
+#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+  /* Don't log early in the bootstrap process, it's normal to pick from a
+   * small pool of nodes. Of course, this won't help if we're trying to
+   * diagnose bootstrap issues. */
+  if (!smartlist_len(nodelist_get_list()) || !usable_consensus
+      || !router_have_minimum_dir_info()) {
+    return;
+  }
+#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */
+
+  /* We couldn't find a node, or the one we have doesn't fit our preferences.
+   * Sometimes this is normal, sometimes it can be a reachability issue. */
+  if (!rs) {
+    /* This happens a lot, so it's at debug level */
+    log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but "
+              "we couldn't find a directory that fit our criteria. "
+              "Perhaps we will succeed next time with less strict criteria.");
+  } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1)
+             && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1)
+             ) {
+    /* This is rare, and might be interesting to users trying to diagnose
+     * connection issues on dual-stack machines. */
+    log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir "
+             "addresses for launching an outgoing connection: "
+             "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d",
+             routerstatus_describe(rs),
+             fmt_addr32(rs->addr), rs->or_port,
+             rs->dir_port, fmt_addr(&rs->ipv6_addr),
+             rs->ipv6_orport, rs->dir_port);
+  }
+}
+
+#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+
+/** How long do we avoid using a directory server after it's given us a 503? */
+#define DIR_503_TIMEOUT (60*60)
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version.
+ * Must be called before RETRY_WITHOUT_EXCLUDE().
+ *
+ * If we got no result, and we are applying IP preferences, and we are a
+ * client that could use an alternate IP version, try again with the
+ * opposite preferences. */
+#define RETRY_ALTERNATE_IP_VERSION(retry_label)                               \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_ip_pref && options->ClientUseIPv4               \
+        && fascist_firewall_use_ipv6(options) && !server_mode(options)        \
+        && !n_busy) {                                                         \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 0;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END                                                                    \
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
+ * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION().
+ *
+ * If we got no result, and we are excluding nodes, and StrictNodes is
+ * not set, try again without excluding nodes. */
+#define RETRY_WITHOUT_EXCLUDE(retry_label)                                    \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_excluding && !options->StrictNodes              \
+        && n_excluded && !n_busy) {                                           \
+      try_excluding = 0;                                                      \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 1;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END
+
+/* Common code used in the loop within router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl.
+ *
+ * Check if the given <b>identity</b> supports extrainfo. If not, skip further
+ * checks.
+ */
+#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity)                        \
+  STMT_BEGIN                                                                  \
+    int is_trusted_extrainfo = router_digest_is_trusted_dir_type(             \
+                               (identity), EXTRAINFO_DIRINFO);                \
+    if (((type) & EXTRAINFO_DIRINFO) &&                                       \
+        !router_supports_extrainfo((identity), is_trusted_extrainfo))         \
+      continue;                                                               \
+  STMT_END
+
+/* When iterating through the routerlist, can OR address/port preference
+ * and reachability checks be skipped?
+ */
+int
+router_skip_or_reachability(const or_options_t *options, int try_ip_pref)
+{
+  /* Servers always have and prefer IPv4.
+   * And if clients are checking against the firewall for reachability only,
+   * but there's no firewall, don't bother checking */
+  return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or());
+}
+
+/* When iterating through the routerlist, can Dir address/port preference
+ * and reachability checks be skipped?
+ */
+static int
+router_skip_dir_reachability(const or_options_t *options, int try_ip_pref)
+{
+  /* Servers always have and prefer IPv4.
+   * And if clients are checking against the firewall for reachability only,
+   * but there's no firewall, don't bother checking */
+  return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir());
+}
+
+/** Pick a random running valid directory server/mirror from our
+ * routerlist.  Arguments are as for router_pick_directory_server(), except:
+ *
+ * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
+ * directories that we excluded for no other reason than
+ * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
+ */
+STATIC const routerstatus_t *
+router_pick_directory_server_impl(dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  const node_t *result;
+  smartlist_t *direct, *tunnel;
+  smartlist_t *trusted_direct, *trusted_tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  time_t now = time(NULL);
+  const networkstatus_t *consensus = networkstatus_get_latest_consensus();
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  int try_excluding = 1, n_excluded = 0, n_busy = 0;
+  int try_ip_pref = 1;
+
+  if (!consensus)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  trusted_direct = smartlist_new();
+  trusted_tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  /* Find all the running dirservers we know about. */
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    int is_trusted;
+    int is_overloaded;
+    const routerstatus_t *status = node->rs;
+    const country_t country = node->country;
+    if (!status)
+      continue;
+
+    if (!node->is_running || !node_is_dir(node) || !node->is_valid)
+      continue;
+    if (requireother && router_digest_is_me(node->identity))
+      continue;
+
+    SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
+
+    if (try_excluding &&
+        routerset_contains_routerstatus(options->ExcludeNodes, status,
+                                        country)) {
+      ++n_excluded;
+      continue;
+    }
+
+    if (router_is_already_dir_fetching_(status->addr,
+                                        &status->ipv6_addr,
+                                        status->dir_port,
+                                        no_serverdesc_fetching,
+                                        no_microdesc_fetching)) {
+      ++n_busy;
+      continue;
+    }
+
+    is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
+    is_trusted = router_digest_is_trusted_dir(node->identity);
+
+    /* Clients use IPv6 addresses if the server has one and the client
+     * prefers IPv6.
+     * Add the router if its preferred address and port are reachable.
+     * If we don't get any routers, we'll try again with the non-preferred
+     * address for each router (if any). (To ensure correct load-balancing
+     * we try routers that only have one address both times.)
+     */
+    if (!fascistfirewall || skip_or_fw ||
+        fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
+                                     try_ip_pref))
+      smartlist_add(is_trusted ? trusted_tunnel :
+                    is_overloaded ? overloaded_tunnel : tunnel, (void*)node);
+    else if (!must_have_or && (skip_dir_fw ||
+             fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION,
+                                          try_ip_pref)))
+      smartlist_add(is_trusted ? trusted_direct :
+                    is_overloaded ? overloaded_direct : direct, (void*)node);
+  } SMARTLIST_FOREACH_END(node);
+
+  if (smartlist_len(tunnel)) {
+    result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_tunnel)) {
+    result = node_sl_choose_by_bandwidth(overloaded_tunnel,
+                                                 WEIGHT_FOR_DIR);
+  } else if (smartlist_len(trusted_tunnel)) {
+    /* FFFF We don't distinguish between trusteds and overloaded trusteds
+     * yet. Maybe one day we should. */
+    /* FFFF We also don't load balance over authorities yet. I think this
+     * is a feature, but it could easily be a bug. -RD */
+    result = smartlist_choose(trusted_tunnel);
+  } else if (smartlist_len(direct)) {
+    result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_direct)) {
+    result = node_sl_choose_by_bandwidth(overloaded_direct,
+                                         WEIGHT_FOR_DIR);
+  } else {
+    result = smartlist_choose(trusted_direct);
+  }
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(trusted_direct);
+  smartlist_free(trusted_tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+
+  router_picked_poor_directory_log(result ? result->rs : NULL);
+
+  return result ? result->rs : NULL;
+}
+
+/** Pick a random element from a list of dir_server_t, weighting by their
+ * <b>weight</b> field. */
+static const dir_server_t *
+dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
+{
+  int n = smartlist_len(servers);
+  int i;
+  double *weights_dbl;
+  uint64_t *weights_u64;
+  const dir_server_t *ds;
+
+  weights_dbl = tor_calloc(n, sizeof(double));
+  weights_u64 = tor_calloc(n, sizeof(uint64_t));
+  for (i = 0; i < n; ++i) {
+    ds = smartlist_get(servers, i);
+    weights_dbl[i] = ds->weight;
+    if (ds->is_authority)
+      weights_dbl[i] *= authority_weight;
+  }
+
+  scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL);
+  i = choose_array_element_by_weight(weights_u64, n);
+  tor_free(weights_dbl);
+  tor_free(weights_u64);
+  return (i < 0) ? NULL : smartlist_get(servers, i);
+}
+
+/** Choose randomly from among the dir_server_ts in sourcelist that
+ * are up. Flags are as for router_pick_directory_server_impl().
+ */
+static const routerstatus_t *
+router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
+                                  dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  smartlist_t *direct, *tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const routerstatus_t *result = NULL;
+  time_t now = time(NULL);
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  const double auth_weight = (sourcelist == fallback_dir_servers) ?
+    options->DirAuthorityFallbackRate : 1.0;
+  smartlist_t *pick_from;
+  int n_busy = 0;
+  int try_excluding = 1, n_excluded = 0;
+  int try_ip_pref = 1;
+
+  if (!sourcelist)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
+    {
+      int is_overloaded =
+          d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
+      if (!d->is_running) continue;
+      if ((type & d->type) == 0)
+        continue;
+
+      SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
+
+      if (requireother && me && router_digest_is_me(d->digest))
+        continue;
+      if (try_excluding &&
+          routerset_contains_routerstatus(options->ExcludeNodes,
+                                          &d->fake_status, -1)) {
+        ++n_excluded;
+        continue;
+      }
+
+      if (router_is_already_dir_fetching_(d->addr,
+                                          &d->ipv6_addr,
+                                          d->dir_port,
+                                          no_serverdesc_fetching,
+                                          no_microdesc_fetching)) {
+        ++n_busy;
+        continue;
+      }
+
+      /* Clients use IPv6 addresses if the server has one and the client
+       * prefers IPv6.
+       * Add the router if its preferred address and port are reachable.
+       * If we don't get any routers, we'll try again with the non-preferred
+       * address for each router (if any). (To ensure correct load-balancing
+       * we try routers that only have one address both times.)
+       */
+      if (!fascistfirewall || skip_or_fw ||
+          fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION,
+                                             try_ip_pref))
+        smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
+      else if (!must_have_or && (skip_dir_fw ||
+               fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION,
+                                                  try_ip_pref)))
+        smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
+    }
+  SMARTLIST_FOREACH_END(d);
+
+  if (smartlist_len(tunnel)) {
+    pick_from = tunnel;
+  } else if (smartlist_len(overloaded_tunnel)) {
+    pick_from = overloaded_tunnel;
+  } else if (smartlist_len(direct)) {
+    pick_from = direct;
+  } else {
+    pick_from = overloaded_direct;
+  }
+
+  {
+    const dir_server_t *selection =
+      dirserver_choose_by_weight(pick_from, auth_weight);
+
+    if (selection)
+      result = &selection->fake_status;
+  }
+
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  router_picked_poor_directory_log(result);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+  return result;
+}
+
+/** Mark as running every dir_server_t in <b>server_list</b>. */
+static void
+mark_all_dirservers_up(smartlist_t *server_list)
+{
+  if (server_list) {
+    SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
+      routerstatus_t *rs;
+      node_t *node;
+      dir->is_running = 1;
+      node = node_get_mutable_by_id(dir->digest);
+      if (node)
+        node->is_running = 1;
+      rs = router_get_mutable_consensus_status_by_id(dir->digest);
+      if (rs) {
+        rs->last_dir_503_at = 0;
+        control_event_networkstatus_changed_single(rs);
+      }
+    } SMARTLIST_FOREACH_END(dir);
+  }
+  router_dir_info_changed();
+}
+
+/** Return true iff r1 and r2 have the same address and OR port. */
+int
+routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2)
+{
+  return r1->addr == r2->addr && r1->or_port == r2->or_port &&
+    tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) &&
+    r1->ipv6_orport == r2->ipv6_orport;
+}
+
+/** Reset all internal variables used to count failed downloads of network
+ * status objects. */
+void
+router_reset_status_download_failures(void)
+{
+  mark_all_dirservers_up(fallback_dir_servers);
+}
+
+/** Given a <b>router</b>, add every node_t in its family (including the
+ * node itself!) to <b>sl</b>.
+ *
+ * Note the type mismatch: This function takes a routerinfo, but adds nodes
+ * to the smartlist!
+ */
+static void
+routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
+{
+  /* XXXX MOVE ? */
+  node_t fake_node;
+  const node_t *node = node_get_by_id(router->cache_info.identity_digest);
+  if (node == NULL) {
+    memset(&fake_node, 0, sizeof(fake_node));
+    fake_node.ri = (routerinfo_t *)router;
+    memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN);
+    node = &fake_node;
+  }
+  nodelist_add_node_and_family(sl, node);
+}
+
+/** Add every suitable node from our nodelist to <b>sl</b>, so that
+ * we can pick a node for a circuit.
+ */
+void
+router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                      int need_capacity, int need_guard,
+                                      int need_desc, int pref_addr,
+                                      int direct_conn)
+{
+  const int check_reach = !router_skip_or_reachability(get_options(),
+                                                       pref_addr);
+  /* XXXX MOVE */
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    if (!node->is_running || !node->is_valid)
+      continue;
+    if (need_desc && !node_has_preferred_descriptor(node, direct_conn))
+      continue;
+    if (node->ri && node->ri->purpose != ROUTER_PURPOSE_GENERAL)
+      continue;
+    if (node_is_unreliable(node, need_uptime, need_capacity, need_guard))
+      continue;
+    /* Don't choose nodes if we are certain they can't do EXTEND2 cells */
+    if (node->rs && !routerstatus_version_supports_extend2_cells(node->rs, 1))
+      continue;
+    /* Don't choose nodes if we are certain they can't do ntor. */
+    if ((node->ri || node->md) && !node_has_curve25519_onion_key(node))
+      continue;
+    /* Choose a node with an OR address that matches the firewall rules */
+    if (direct_conn && check_reach &&
+        !fascist_firewall_allows_node(node,
+                                      FIREWALL_OR_CONNECTION,
+                                      pref_addr))
+      continue;
+
+    smartlist_add(sl, (void *)node);
+  } SMARTLIST_FOREACH_END(node);
+}
+
+/** Look through the routerlist until we find a router that has my key.
+ Return it. */
+const routerinfo_t *
+routerlist_find_my_routerinfo(void)
+{
+  if (!routerlist)
+    return NULL;
+
+  SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, router,
+  {
+    if (router_is_me(router))
+      return router;
+  });
+  return NULL;
+}
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity. */
+uint32_t
+router_get_advertised_bandwidth(const routerinfo_t *router)
+{
+  if (router->bandwidthcapacity < router->bandwidthrate)
+    return router->bandwidthcapacity;
+  return router->bandwidthrate;
+}
+
+/** Do not weight any declared bandwidth more than this much when picking
+ * routers by bandwidth. */
+#define DEFAULT_MAX_BELIEVABLE_BANDWIDTH 10000000 /* 10 MB/sec */
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity, capped by max-believe-bw. */
+uint32_t
+router_get_advertised_bandwidth_capped(const routerinfo_t *router)
+{
+  uint32_t result = router->bandwidthcapacity;
+  if (result > router->bandwidthrate)
+    result = router->bandwidthrate;
+  if (result > DEFAULT_MAX_BELIEVABLE_BANDWIDTH)
+    result = DEFAULT_MAX_BELIEVABLE_BANDWIDTH;
+  return result;
+}
+
+/** Given an array of double/uint64_t unions that are currently being used as
+ * doubles, convert them to uint64_t, and try to scale them linearly so as to
+ * much of the range of uint64_t. If <b>total_out</b> is provided, set it to
+ * the sum of all elements in the array _before_ scaling. */
+STATIC void
+scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in,
+                            int n_entries,
+                            uint64_t *total_out)
+{
+  double total = 0.0;
+  double scale_factor = 0.0;
+  int i;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries_in[i];
+
+  if (total > 0.0) {
+    scale_factor = ((double)INT64_MAX) / total;
+    scale_factor /= 4.0; /* make sure we're very far away from overflowing */
+  }
+
+  for (i = 0; i < n_entries; ++i)
+    entries_out[i] = tor_llround(entries_in[i] * scale_factor);
+
+  if (total_out)
+    *total_out = (uint64_t) total;
+}
+
+/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>,
+ * choosing each element with a probability proportional to its (uint64_t)
+ * value, and return the index of that element.  If all elements are 0, choose
+ * an index at random. Return -1 on error.
+ */
+STATIC int
+choose_array_element_by_weight(const uint64_t *entries, int n_entries)
+{
+  int i;
+  uint64_t rand_val;
+  uint64_t total = 0;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries[i];
+
+  if (n_entries < 1)
+    return -1;
+
+  if (total == 0)
+    return crypto_rand_int(n_entries);
+
+  tor_assert(total < INT64_MAX);
+
+  rand_val = crypto_rand_uint64(total);
+
+  return select_array_member_cumulative_timei(
+                           entries, n_entries, total, rand_val);
+}
+
+/** When weighting bridges, enforce these values as lower and upper
+ * bound for believable bandwidth, because there is no way for us
+ * to verify a bridge's bandwidth currently. */
+#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000  /* 20 kB/sec */
+#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity, making sure to stay within the
+ * interval between bridge-min-believe-bw and
+ * bridge-max-believe-bw. */
+static uint32_t
+bridge_get_advertised_bandwidth_bounded(routerinfo_t *router)
+{
+  uint32_t result = router->bandwidthcapacity;
+  if (result > router->bandwidthrate)
+    result = router->bandwidthrate;
+  if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH;
+  else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH;
+  return result;
+}
+
+/** Return bw*1000, unless bw*1000 would overflow, in which case return
+ * INT32_MAX. */
+static inline int32_t
+kb_to_bytes(uint32_t bw)
+{
+  return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
+}
+
+/** Helper function:
+ * choose a random element of smartlist <b>sl</b> of nodes, weighted by
+ * the advertised bandwidth of each element using the consensus
+ * bandwidth weights.
+ *
+ * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all
+ * nodes' bandwidth equally regardless of their Exit status, since there may
+ * be some in the list because they exit to obscure ports. If
+ * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight
+ * exit-node's bandwidth less depending on the smallness of the fraction of
+ * Exit-to-total bandwidth.  If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a
+ * guard node: consider all guard's bandwidth equally. Otherwise, weight
+ * guards proportionally less.
+ */
+static const node_t *
+smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl,
+                                           bandwidth_weight_rule_t rule)
+{
+  double *bandwidths_dbl=NULL;
+  uint64_t *bandwidths_u64=NULL;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0)
+    return NULL;
+
+  bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t));
+  scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl,
+                              smartlist_len(sl), NULL);
+
+  {
+    int idx = choose_array_element_by_weight(bandwidths_u64,
+                                             smartlist_len(sl));
+    tor_free(bandwidths_dbl);
+    tor_free(bandwidths_u64);
+    return idx < 0 ? NULL : smartlist_get(sl, idx);
+  }
+}
+
+/** Given a list of routers and a weighting rule as in
+ * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth
+ * values for each node and store them in a freshly allocated
+ * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results
+ * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total
+ * of all the bandwidths.
+ * Return 0 on success, -1 on failure. */
+static int
+compute_weighted_bandwidths(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            double **bandwidths_out,
+                            double *total_bandwidth_out)
+{
+  int64_t weight_scale;
+  double Wg = -1, Wm = -1, We = -1, Wd = -1;
+  double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1;
+  guardfraction_bandwidth_t guardfraction_bw;
+  double *bandwidths = NULL;
+  double total_bandwidth = 0.0;
+
+  tor_assert(sl);
+  tor_assert(bandwidths_out);
+
+  /* Can't choose exit and guard at same time */
+  tor_assert(rule == NO_WEIGHTING ||
+             rule == WEIGHT_FOR_EXIT ||
+             rule == WEIGHT_FOR_GUARD ||
+             rule == WEIGHT_FOR_MID ||
+             rule == WEIGHT_FOR_DIR);
+
+  *bandwidths_out = NULL;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = 0.0;
+  }
+
+  if (smartlist_len(sl) == 0) {
+    log_info(LD_CIRC,
+             "Empty routerlist passed in to consensus weight node "
+             "selection for rule %s",
+             bandwidth_weight_rule_to_string(rule));
+    return -1;
+  }
+
+  weight_scale = networkstatus_get_weight_scale_param(NULL);
+
+  if (rule == WEIGHT_FOR_GUARD) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */
+    We = 0;
+    Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_MID) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1);
+    We = networkstatus_get_bw_weight(NULL, "Wme", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_EXIT) {
+    // Guards CAN be exits if they have weird exit policies
+    // They are d then I guess...
+    We = networkstatus_get_bw_weight(NULL, "Wee", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */
+    Wd = networkstatus_get_bw_weight(NULL, "Wed", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_DIR) {
+    We = networkstatus_get_bw_weight(NULL, "Wbe", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1);
+
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  } else if (rule == NO_WEIGHTING) {
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0
+      || Web < 0) {
+    log_debug(LD_CIRC,
+              "Got negative bandwidth weights. Defaulting to naive selection"
+              " algorithm.");
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  Wg /= weight_scale;
+  Wm /= weight_scale;
+  We /= weight_scale;
+  Wd /= weight_scale;
+
+  Wgb /= weight_scale;
+  Wmb /= weight_scale;
+  Web /= weight_scale;
+  Wdb /= weight_scale;
+
+  bandwidths = tor_calloc(smartlist_len(sl), sizeof(double));
+
+  // Cycle through smartlist and total the bandwidth.
+  static int warned_missing_bw = 0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0;
+    double weight = 1;
+    double weight_without_guard_flag = 0; /* Used for guardfraction */
+    double final_weight = 0;
+    is_exit = node->is_exit && ! node->is_bad_exit;
+    is_guard = node->is_possible_guard;
+    is_dir = node_is_dir(node);
+    if (node->rs) {
+      if (!node->rs->has_bandwidth) {
+        /* This should never happen, unless all the authorities downgrade
+         * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
+        if (! warned_missing_bw) {
+          log_warn(LD_BUG,
+                 "Consensus is missing some bandwidths. Using a naive "
+                 "router selection algorithm");
+          warned_missing_bw = 1;
+        }
+        this_bw = 30000; /* Chosen arbitrarily */
+      } else {
+        this_bw = kb_to_bytes(node->rs->bandwidth_kb);
+      }
+    } else if (node->ri) {
+      /* bridge or other descriptor not in our consensus */
+      this_bw = bridge_get_advertised_bandwidth_bounded(node->ri);
+    } else {
+      /* We can't use this one. */
+      continue;
+    }
+
+    if (is_guard && is_exit) {
+      weight = (is_dir ? Wdb*Wd : Wd);
+      weight_without_guard_flag = (is_dir ? Web*We : We);
+    } else if (is_guard) {
+      weight = (is_dir ? Wgb*Wg : Wg);
+      weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm);
+    } else if (is_exit) {
+      weight = (is_dir ? Web*We : We);
+    } else { // middle
+      weight = (is_dir ? Wmb*Wm : Wm);
+    }
+    /* These should be impossible; but overflows here would be bad, so let's
+     * make sure. */
+    if (this_bw < 0)
+      this_bw = 0;
+    if (weight < 0.0)
+      weight = 0.0;
+    if (weight_without_guard_flag < 0.0)
+      weight_without_guard_flag = 0.0;
+
+    /* If guardfraction information is available in the consensus, we
+     * want to calculate this router's bandwidth according to its
+     * guardfraction. Quoting from proposal236:
+     *
+     *    Let Wpf denote the weight from the 'bandwidth-weights' line a
+     *    client would apply to N for position p if it had the guard
+     *    flag, Wpn the weight if it did not have the guard flag, and B the
+     *    measured bandwidth of N in the consensus.  Then instead of choosing
+     *    N for position p proportionally to Wpf*B or Wpn*B, clients should
+     *    choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+     */
+    if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) {
+      /* XXX The assert should actually check for is_guard. However,
+       * that crashes dirauths because of #13297. This should be
+       * equivalent: */
+      tor_assert(node->rs->is_possible_guard);
+
+      guard_get_guardfraction_bandwidth(&guardfraction_bw,
+                                        this_bw,
+                                        node->rs->guardfraction_percentage);
+
+      /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */
+      final_weight =
+        guardfraction_bw.guard_bw * weight +
+        guardfraction_bw.non_guard_bw * weight_without_guard_flag;
+
+      log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)",
+                node->rs->nickname, final_weight, weight*this_bw,
+                bandwidth_weight_rule_to_string(rule));
+    } else { /* no guardfraction information. calculate the weight normally. */
+      final_weight = weight*this_bw;
+    }
+
+    bandwidths[node_sl_idx] = final_weight;
+    total_bandwidth += final_weight;
+  } SMARTLIST_FOREACH_END(node);
+
+  log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based "
+            "on weights "
+            "Wg=%f Wm=%f We=%f Wd=%f with total bw %f",
+            bandwidth_weight_rule_to_string(rule),
+            Wg, Wm, We, Wd, total_bandwidth);
+
+  *bandwidths_out = bandwidths;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = total_bandwidth;
+  }
+
+  return 0;
+}
+
+/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted
+ * by their weighted bandwidths with rule <b>rule</b>, for which we have
+ * descriptors.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect
+ * to it indirectly, or use it as if we were connecting to it indirectly. */
+double
+frac_nodes_with_descriptors(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            int for_direct_conn)
+{
+  double *bandwidths = NULL;
+  double total, present;
+
+  if (smartlist_len(sl) == 0)
+    return 0.0;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 ||
+      total <= 0.0) {
+    int n_with_descs = 0;
+    SMARTLIST_FOREACH(sl, const node_t *, node, {
+      if (node_has_preferred_descriptor(node, for_direct_conn))
+        n_with_descs++;
+    });
+    tor_free(bandwidths);
+    return ((double)n_with_descs) / smartlist_len(sl);
+  }
+
+  present = 0.0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    if (node_has_preferred_descriptor(node, for_direct_conn))
+      present += bandwidths[node_sl_idx];
+  } SMARTLIST_FOREACH_END(node);
+
+  tor_free(bandwidths);
+
+  return present / total;
+}
+
+/** Choose a random element of status list <b>sl</b>, weighted by
+ * the advertised bandwidth of each node */
+const node_t *
+node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule)
+{ /*XXXX MOVE */
+  return smartlist_choose_node_by_bandwidth_weights(sl, rule);
+}
+
+/** Return a random running node from the nodelist. Never
+ * pick a node that is in
+ * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
+ * even if they are the only nodes available.
+ * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than
+ * a minimum uptime, return one of those.
+ * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
+ * advertised capacity of each router.
+ * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
+ * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
+ * picking an exit node, otherwise we weight bandwidths for picking a relay
+ * node (that is, possibly discounting exit nodes).
+ * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that
+ * have a routerinfo or microdescriptor -- that is, enough info to be
+ * used to build a circuit.
+ * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that
+ * have an address that is preferred by the ClientPreferIPv6ORPort setting
+ * (regardless of this flag, we exclude nodes that aren't allowed by the
+ * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0).
+ */
+const node_t *
+router_choose_random_node(smartlist_t *excludedsmartlist,
+                          routerset_t *excludedset,
+                          router_crn_flags_t flags)
+{ /* XXXX MOVE */
+  const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
+  const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
+  const int need_guard = (flags & CRN_NEED_GUARD) != 0;
+  const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
+  const int need_desc = (flags & CRN_NEED_DESC) != 0;
+  const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
+  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
+  const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
+
+  smartlist_t *sl=smartlist_new(),
+    *excludednodes=smartlist_new();
+  const node_t *choice = NULL;
+  const routerinfo_t *r;
+  bandwidth_weight_rule_t rule;
+
+  tor_assert(!(weight_for_exit && need_guard));
+  rule = weight_for_exit ? WEIGHT_FOR_EXIT :
+    (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
+
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+    if (node_allows_single_hop_exits(node)) {
+      /* Exclude relays that allow single hop exit circuits. This is an
+       * obsolete option since 0.2.9.2-alpha and done by default in
+       * 0.3.1.0-alpha. */
+      smartlist_add(excludednodes, node);
+    } else if (rendezvous_v3 &&
+               !node_supports_v3_rendezvous_point(node)) {
+      /* Exclude relays that do not support to rendezvous for a hidden service
+       * version 3. */
+      smartlist_add(excludednodes, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* If the node_t is not found we won't be to exclude ourself but we
+   * won't be able to pick ourself in router_choose_random_node() so
+   * this is fine to at least try with our routerinfo_t object. */
+  if ((r = router_get_my_routerinfo()))
+    routerlist_add_node_and_family(excludednodes, r);
+
+  router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
+                                        need_guard, need_desc, pref_addr,
+                                        direct_conn);
+  log_debug(LD_CIRC,
+           "We found %d running nodes.",
+            smartlist_len(sl));
+
+  smartlist_subtract(sl,excludednodes);
+  log_debug(LD_CIRC,
+            "We removed %d excludednodes, leaving %d nodes.",
+            smartlist_len(excludednodes),
+            smartlist_len(sl));
+
+  if (excludedsmartlist) {
+    smartlist_subtract(sl,excludedsmartlist);
+    log_debug(LD_CIRC,
+              "We removed %d excludedsmartlist, leaving %d nodes.",
+              smartlist_len(excludedsmartlist),
+              smartlist_len(sl));
+  }
+  if (excludedset) {
+    routerset_subtract_nodes(sl,excludedset);
+    log_debug(LD_CIRC,
+              "We removed excludedset, leaving %d nodes.",
+              smartlist_len(sl));
+  }
+
+  // Always weight by bandwidth
+  choice = node_sl_choose_by_bandwidth(sl, rule);
+
+  smartlist_free(sl);
+  if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) {
+    /* try once more -- recurse but with fewer restrictions. */
+    log_info(LD_CIRC,
+             "We couldn't find any live%s%s%s routers; falling back "
+             "to list of all routers.",
+             need_capacity?", fast":"",
+             need_uptime?", stable":"",
+             need_guard?", guard":"");
+    flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD|
+                CRN_PREF_ADDR);
+    choice = router_choose_random_node(
+                     excludedsmartlist, excludedset, flags);
+  }
+  smartlist_free(excludednodes);
+  if (!choice) {
+    log_warn(LD_CIRC,
+             "No available nodes when trying to choose node. Failing.");
+  }
+  return choice;
+}
+
+/** Helper: given an extended nickname in <b>hexdigest</b> try to decode it.
+ * Return 0 on success, -1 on failure.  Store the result into the
+ * DIGEST_LEN-byte buffer at <b>digest_out</b>, the single character at
+ * <b>nickname_qualifier_char_out</b>, and the MAXNICKNAME_LEN+1-byte buffer
+ * at <b>nickname_out</b>.
+ *
+ * The recognized format is:
+ *   HexName = Dollar? HexDigest NamePart?
+ *   Dollar = '?'
+ *   HexDigest = HexChar*20
+ *   HexChar = 'a'..'f' | 'A'..'F' | '0'..'9'
+ *   NamePart = QualChar Name
+ *   QualChar = '=' | '~'
+ *   Name = NameChar*(1..MAX_NICKNAME_LEN)
+ *   NameChar = Any ASCII alphanumeric character
+ */
+int
+hex_digest_nickname_decode(const char *hexdigest,
+                           char *digest_out,
+                           char *nickname_qualifier_char_out,
+                           char *nickname_out)
+{
+  size_t len;
+
+  tor_assert(hexdigest);
+  if (hexdigest[0] == '$')
+    ++hexdigest;
+
+  len = strlen(hexdigest);
+  if (len < HEX_DIGEST_LEN) {
+    return -1;
+  } else if (len > HEX_DIGEST_LEN && (hexdigest[HEX_DIGEST_LEN] == '=' ||
+                                    hexdigest[HEX_DIGEST_LEN] == '~') &&
+           len <= HEX_DIGEST_LEN+1+MAX_NICKNAME_LEN) {
+    *nickname_qualifier_char_out = hexdigest[HEX_DIGEST_LEN];
+    strlcpy(nickname_out, hexdigest+HEX_DIGEST_LEN+1 , MAX_NICKNAME_LEN+1);
+  } else if (len == HEX_DIGEST_LEN) {
+    ;
+  } else {
+    return -1;
+  }
+
+  if (base16_decode(digest_out, DIGEST_LEN,
+                    hexdigest, HEX_DIGEST_LEN) != DIGEST_LEN)
+    return -1;
+  return 0;
+}
+
+/** Helper: Return true iff the <b>identity_digest</b> and <b>nickname</b>
+ * combination of a router, encoded in hexadecimal, matches <b>hexdigest</b>
+ * (which is optionally prefixed with a single dollar sign).  Return false if
+ * <b>hexdigest</b> is malformed, or it doesn't match.  */
+int
+hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest,
+                            const char *nickname)
+{
+  char digest[DIGEST_LEN];
+  char nn_char='\0';
+  char nn_buf[MAX_NICKNAME_LEN+1];
+
+  if (hex_digest_nickname_decode(hexdigest, digest, &nn_char, nn_buf) == -1)
+    return 0;
+
+  if (nn_char == '=') {
+    return 0;
+  }
+
+  if (nn_char == '~') {
+    if (!nickname)     // XXX This seems wrong. -NM
+      return 0;
+    if (strcasecmp(nn_buf, nickname))
+      return 0;
+  }
+
+  return tor_memeq(digest, identity_digest, DIGEST_LEN);
+}
+
+/** Return true iff <b>digest</b> is the digest of the identity key of a
+ * trusted directory matching at least one bit of <b>type</b>.  If <b>type</b>
+ * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
+int
+router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
+{
+  if (!trusted_dir_servers)
+    return 0;
+  if (authdir_mode(get_options()) && router_digest_is_me(digest))
+    return 1;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
+    if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
+      return (!type) || ((type & ent->type) != 0);
+    });
+  return 0;
+}
+
+/** If hexdigest is correctly formed, base16_decode it into
+ * digest, which must have DIGEST_LEN space in it.
+ * Return 0 on success, -1 on failure.
+ */
+int
+hexdigest_to_digest(const char *hexdigest, char *digest)
+{
+  if (hexdigest[0]=='$')
+    ++hexdigest;
+  if (strlen(hexdigest) < HEX_DIGEST_LEN ||
+      base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN)
+    return -1;
+  return 0;
+}
+
+/** As router_get_by_id_digest,but return a pointer that you're allowed to
+ * modify */
+routerinfo_t *
+router_get_mutable_by_digest(const char *digest)
+{
+  tor_assert(digest);
+
+  if (!routerlist) return NULL;
+
+  // routerlist_assert_ok(routerlist);
+
+  return rimap_get(routerlist->identity_map, digest);
+}
+
+/** Return the router in our routerlist whose 20-byte key digest
+ * is <b>digest</b>.  Return NULL if no such router is known. */
+const routerinfo_t *
+router_get_by_id_digest(const char *digest)
+{
+  return router_get_mutable_by_digest(digest);
+}
+
+/** Return the router in our routerlist whose 20-byte descriptor
+ * is <b>digest</b>.  Return NULL if no such router is known. */
+signed_descriptor_t *
+router_get_by_descriptor_digest(const char *digest)
+{
+  tor_assert(digest);
+
+  if (!routerlist) return NULL;
+
+  return sdmap_get(routerlist->desc_digest_map, digest);
+}
+
+/** Return the signed descriptor for the router in our routerlist whose
+ * 20-byte extra-info digest is <b>digest</b>.  Return NULL if no such router
+ * is known. */
+MOCK_IMPL(signed_descriptor_t *,
+router_get_by_extrainfo_digest,(const char *digest))
+{
+  tor_assert(digest);
+
+  if (!routerlist) return NULL;
+
+  return sdmap_get(routerlist->desc_by_eid_map, digest);
+}
+
+/** Return the signed descriptor for the extrainfo_t in our routerlist whose
+ * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info
+ * document is known. */
+MOCK_IMPL(signed_descriptor_t *,
+extrainfo_get_by_descriptor_digest,(const char *digest))
+{
+  extrainfo_t *ei;
+  tor_assert(digest);
+  if (!routerlist) return NULL;
+  ei = eimap_get(routerlist->extra_info_map, digest);
+  return ei ? &ei->cache_info : NULL;
+}
+
+/** Return a pointer to the signed textual representation of a descriptor.
+ * The returned string is not guaranteed to be NUL-terminated: the string's
+ * length will be in desc-\>signed_descriptor_len.
+ *
+ * If <b>with_annotations</b> is set, the returned string will include
+ * the annotations
+ * (if any) preceding the descriptor.  This will increase the length of the
+ * string by desc-\>annotations_len.
+ *
+ * The caller must not free the string returned.
+ */
+static const char *
+signed_descriptor_get_body_impl(const signed_descriptor_t *desc,
+                                int with_annotations)
+{
+  const char *r = NULL;
+  size_t len = desc->signed_descriptor_len;
+  off_t offset = desc->saved_offset;
+  if (with_annotations)
+    len += desc->annotations_len;
+  else
+    offset += desc->annotations_len;
+
+  tor_assert(len > 32);
+  if (desc->saved_location == SAVED_IN_CACHE && routerlist) {
+    desc_store_t *store = desc_get_store(router_get_routerlist(), desc);
+    if (store && store->mmap) {
+      tor_assert(desc->saved_offset + len <= store->mmap->size);
+      r = store->mmap->data + offset;
+    } else if (store) {
+      log_err(LD_DIR, "We couldn't read a descriptor that is supposedly "
+              "mmaped in our cache.  Is another process running in our data "
+              "directory?  Exiting.");
+      exit(1); // XXXX bad exit: should recover.
+    }
+  }
+  if (!r) /* no mmap, or not in cache. */
+    r = desc->signed_descriptor_body +
+      (with_annotations ? 0 : desc->annotations_len);
+
+  tor_assert(r);
+  if (!with_annotations) {
+    if (fast_memcmp("router ", r, 7) && fast_memcmp("extra-info ", r, 11)) {
+      char *cp = tor_strndup(r, 64);
+      log_err(LD_DIR, "descriptor at %p begins with unexpected string %s.  "
+              "Is another process running in our data directory?  Exiting.",
+              desc, escaped(cp));
+      exit(1); // XXXX bad exit: should recover.
+    }
+  }
+
+  return r;
+}
+
+/** Return a pointer to the signed textual representation of a descriptor.
+ * The returned string is not guaranteed to be NUL-terminated: the string's
+ * length will be in desc-\>signed_descriptor_len.
+ *
+ * The caller must not free the string returned.
+ */
+const char *
+signed_descriptor_get_body(const signed_descriptor_t *desc)
+{
+  return signed_descriptor_get_body_impl(desc, 0);
+}
+
+/** As signed_descriptor_get_body(), but points to the beginning of the
+ * annotations section rather than the beginning of the descriptor. */
+const char *
+signed_descriptor_get_annotations(const signed_descriptor_t *desc)
+{
+  return signed_descriptor_get_body_impl(desc, 1);
+}
+
+/** Return the current list of all known routers. */
+routerlist_t *
+router_get_routerlist(void)
+{
+  if (PREDICT_UNLIKELY(!routerlist)) {
+    routerlist = tor_malloc_zero(sizeof(routerlist_t));
+    routerlist->routers = smartlist_new();
+    routerlist->old_routers = smartlist_new();
+    routerlist->identity_map = rimap_new();
+    routerlist->desc_digest_map = sdmap_new();
+    routerlist->desc_by_eid_map = sdmap_new();
+    routerlist->extra_info_map = eimap_new();
+
+    routerlist->desc_store.fname_base = "cached-descriptors";
+    routerlist->extrainfo_store.fname_base = "cached-extrainfo";
+
+    routerlist->desc_store.type = ROUTER_STORE;
+    routerlist->extrainfo_store.type = EXTRAINFO_STORE;
+
+    routerlist->desc_store.description = "router descriptors";
+    routerlist->extrainfo_store.description = "extra-info documents";
+  }
+  return routerlist;
+}
+
+/** Free all storage held by <b>router</b>. */
+void
+routerinfo_free_(routerinfo_t *router)
+{
+  if (!router)
+    return;
+
+  tor_free(router->cache_info.signed_descriptor_body);
+  tor_free(router->nickname);
+  tor_free(router->platform);
+  tor_free(router->protocol_list);
+  tor_free(router->contact_info);
+  if (router->onion_pkey)
+    crypto_pk_free(router->onion_pkey);
+  tor_free(router->onion_curve25519_pkey);
+  if (router->identity_pkey)
+    crypto_pk_free(router->identity_pkey);
+  tor_cert_free(router->cache_info.signing_key_cert);
+  if (router->declared_family) {
+    SMARTLIST_FOREACH(router->declared_family, char *, s, tor_free(s));
+    smartlist_free(router->declared_family);
+  }
+  addr_policy_list_free(router->exit_policy);
+  short_policy_free(router->ipv6_exit_policy);
+
+  memset(router, 77, sizeof(routerinfo_t));
+
+  tor_free(router);
+}
+
+/** Release all storage held by <b>extrainfo</b> */
+void
+extrainfo_free_(extrainfo_t *extrainfo)
+{
+  if (!extrainfo)
+    return;
+  tor_cert_free(extrainfo->cache_info.signing_key_cert);
+  tor_free(extrainfo->cache_info.signed_descriptor_body);
+  tor_free(extrainfo->pending_sig);
+
+  memset(extrainfo, 88, sizeof(extrainfo_t)); /* debug bad memory usage */
+  tor_free(extrainfo);
+}
+
+#define signed_descriptor_free(val) \
+  FREE_AND_NULL(signed_descriptor_t, signed_descriptor_free_, (val))
+
+/** Release storage held by <b>sd</b>. */
+static void
+signed_descriptor_free_(signed_descriptor_t *sd)
+{
+  if (!sd)
+    return;
+
+  tor_free(sd->signed_descriptor_body);
+  tor_cert_free(sd->signing_key_cert);
+
+  memset(sd, 99, sizeof(signed_descriptor_t)); /* Debug bad mem usage */
+  tor_free(sd);
+}
+
+/** Reset the given signed descriptor <b>sd</b> by freeing the allocated
+ * memory inside the object and by zeroing its content. */
+static void
+signed_descriptor_reset(signed_descriptor_t *sd)
+{
+  tor_assert(sd);
+  tor_free(sd->signed_descriptor_body);
+  tor_cert_free(sd->signing_key_cert);
+  memset(sd, 0, sizeof(*sd));
+}
+
+/** Copy src into dest, and steal all references inside src so that when
+ * we free src, we don't mess up dest. */
+static void
+signed_descriptor_move(signed_descriptor_t *dest,
+                       signed_descriptor_t *src)
+{
+  tor_assert(dest != src);
+  /* Cleanup destination object before overwriting it.*/
+  signed_descriptor_reset(dest);
+  memcpy(dest, src, sizeof(signed_descriptor_t));
+  src->signed_descriptor_body = NULL;
+  src->signing_key_cert = NULL;
+  dest->routerlist_index = -1;
+}
+
+/** Extract a signed_descriptor_t from a general routerinfo, and free the
+ * routerinfo.
+ */
+static signed_descriptor_t *
+signed_descriptor_from_routerinfo(routerinfo_t *ri)
+{
+  signed_descriptor_t *sd;
+  tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL);
+  sd = tor_malloc_zero(sizeof(signed_descriptor_t));
+  signed_descriptor_move(sd, &ri->cache_info);
+  routerinfo_free(ri);
+  return sd;
+}
+
+/** Helper: free the storage held by the extrainfo_t in <b>e</b>. */
+static void
+extrainfo_free_void(void *e)
+{
+  extrainfo_free_(e);
+}
+
+/** Free all storage held by a routerlist <b>rl</b>. */
+void
+routerlist_free_(routerlist_t *rl)
+{
+  if (!rl)
+    return;
+  rimap_free(rl->identity_map, NULL);
+  sdmap_free(rl->desc_digest_map, NULL);
+  sdmap_free(rl->desc_by_eid_map, NULL);
+  eimap_free(rl->extra_info_map, extrainfo_free_void);
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
+                    routerinfo_free(r));
+  SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd,
+                    signed_descriptor_free(sd));
+  smartlist_free(rl->routers);
+  smartlist_free(rl->old_routers);
+  if (rl->desc_store.mmap) {
+    int res = tor_munmap_file(routerlist->desc_store.mmap);
+    if (res != 0) {
+      log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap");
+    }
+  }
+  if (rl->extrainfo_store.mmap) {
+    int res = tor_munmap_file(routerlist->extrainfo_store.mmap);
+    if (res != 0) {
+      log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap");
+    }
+  }
+  tor_free(rl);
+
+  router_dir_info_changed();
+}
+
+/** Log information about how much memory is being used for routerlist,
+ * at log level <b>severity</b>. */
+void
+dump_routerlist_mem_usage(int severity)
+{
+  uint64_t livedescs = 0;
+  uint64_t olddescs = 0;
+  if (!routerlist)
+    return;
+  SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, r,
+                    livedescs += r->cache_info.signed_descriptor_len);
+  SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd,
+                    olddescs += sd->signed_descriptor_len);
+
+  tor_log(severity, LD_DIR,
+      "In %d live descriptors: %"PRIu64" bytes.  "
+      "In %d old descriptors: %"PRIu64" bytes.",
+      smartlist_len(routerlist->routers), (livedescs),
+      smartlist_len(routerlist->old_routers), (olddescs));
+}
+
+/** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is
+ * in <b>sl</b> at position <b>idx</b>. Otherwise, search <b>sl</b> for
+ * <b>ri</b>.  Return the index of <b>ri</b> in <b>sl</b>, or -1 if <b>ri</b>
+ * is not in <b>sl</b>. */
+static inline int
+routerlist_find_elt_(smartlist_t *sl, void *ri, int idx)
+{
+  if (idx < 0) {
+    idx = -1;
+    SMARTLIST_FOREACH(sl, routerinfo_t *, r,
+                      if (r == ri) {
+                        idx = r_sl_idx;
+                        break;
+                      });
+  } else {
+    tor_assert(idx < smartlist_len(sl));
+    tor_assert(smartlist_get(sl, idx) == ri);
+  };
+  return idx;
+}
+
+/** Insert an item <b>ri</b> into the routerlist <b>rl</b>, updating indices
+ * as needed.  There must be no previous member of <b>rl</b> with the same
+ * identity digest as <b>ri</b>: If there is, call routerlist_replace
+ * instead.
+ */
+static void
+routerlist_insert(routerlist_t *rl, routerinfo_t *ri)
+{
+  routerinfo_t *ri_old;
+  signed_descriptor_t *sd_old;
+  {
+    const routerinfo_t *ri_generated = router_get_my_routerinfo();
+    tor_assert(ri_generated != ri);
+  }
+  tor_assert(ri->cache_info.routerlist_index == -1);
+
+  ri_old = rimap_set(rl->identity_map, ri->cache_info.identity_digest, ri);
+  tor_assert(!ri_old);
+
+  sd_old = sdmap_set(rl->desc_digest_map,
+                     ri->cache_info.signed_descriptor_digest,
+                     &(ri->cache_info));
+  if (sd_old) {
+    int idx = sd_old->routerlist_index;
+    sd_old->routerlist_index = -1;
+    smartlist_del(rl->old_routers, idx);
+    if (idx < smartlist_len(rl->old_routers)) {
+       signed_descriptor_t *d = smartlist_get(rl->old_routers, idx);
+       d->routerlist_index = idx;
+    }
+    rl->desc_store.bytes_dropped += sd_old->signed_descriptor_len;
+    sdmap_remove(rl->desc_by_eid_map, sd_old->extra_info_digest);
+    signed_descriptor_free(sd_old);
+  }
+
+  if (!tor_digest_is_zero(ri->cache_info.extra_info_digest))
+    sdmap_set(rl->desc_by_eid_map, ri->cache_info.extra_info_digest,
+              &ri->cache_info);
+  smartlist_add(rl->routers, ri);
+  ri->cache_info.routerlist_index = smartlist_len(rl->routers) - 1;
+  nodelist_set_routerinfo(ri, NULL);
+  router_dir_info_changed();
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Adds the extrainfo_t <b>ei</b> to the routerlist <b>rl</b>, if there is a
+ * corresponding router in rl-\>routers or rl-\>old_routers.  Return the status
+ * of inserting <b>ei</b>.  Free <b>ei</b> if it isn't inserted. */
+MOCK_IMPL(STATIC was_router_added_t,
+extrainfo_insert,(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible))
+{
+  was_router_added_t r;
+  const char *compatibility_error_msg;
+  routerinfo_t *ri = rimap_get(rl->identity_map,
+                               ei->cache_info.identity_digest);
+  signed_descriptor_t *sd =
+    sdmap_get(rl->desc_by_eid_map, ei->cache_info.signed_descriptor_digest);
+  extrainfo_t *ei_tmp;
+  const int severity = warn_if_incompatible ? LOG_WARN : LOG_INFO;
+
+  {
+    extrainfo_t *ei_generated = router_get_my_extrainfo();
+    tor_assert(ei_generated != ei);
+  }
+
+  if (!ri) {
+    /* This router is unknown; we can't even verify the signature. Give up.*/
+    r = ROUTER_NOT_IN_CONSENSUS;
+    goto done;
+  }
+  if (! sd) {
+    /* The extrainfo router doesn't have a known routerdesc to attach it to.
+     * This just won't work. */;
+    static ratelim_t no_sd_ratelim = RATELIM_INIT(1800);
+    r = ROUTER_BAD_EI;
+    log_fn_ratelim(&no_sd_ratelim, severity, LD_BUG,
+                   "No entry found in extrainfo map.");
+    goto done;
+  }
+  if (tor_memneq(ei->cache_info.signed_descriptor_digest,
+                 sd->extra_info_digest, DIGEST_LEN)) {
+    static ratelim_t digest_mismatch_ratelim = RATELIM_INIT(1800);
+    /* The sd we got from the map doesn't match the digest we used to look
+     * it up. This makes no sense. */
+    r = ROUTER_BAD_EI;
+    log_fn_ratelim(&digest_mismatch_ratelim, severity, LD_BUG,
+                     "Mismatch in digest in extrainfo map.");
+    goto done;
+  }
+  if (routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, sd,
+                                             &compatibility_error_msg)) {
+    char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1];
+    r = (ri->cache_info.extrainfo_is_bogus) ?
+      ROUTER_BAD_EI : ROUTER_NOT_IN_CONSENSUS;
+
+    base16_encode(d1, sizeof(d1), ri->cache_info.identity_digest, DIGEST_LEN);
+    base16_encode(d2, sizeof(d2), ei->cache_info.identity_digest, DIGEST_LEN);
+
+    log_fn(severity,LD_DIR,
+           "router info incompatible with extra info (ri id: %s, ei id %s, "
+           "reason: %s)", d1, d2, compatibility_error_msg);
+
+    goto done;
+  }
+
+  /* Okay, if we make it here, we definitely have a router corresponding to
+   * this extrainfo. */
+
+  ei_tmp = eimap_set(rl->extra_info_map,
+                     ei->cache_info.signed_descriptor_digest,
+                     ei);
+  r = ROUTER_ADDED_SUCCESSFULLY;
+  if (ei_tmp) {
+    rl->extrainfo_store.bytes_dropped +=
+      ei_tmp->cache_info.signed_descriptor_len;
+    extrainfo_free(ei_tmp);
+  }
+
+ done:
+  if (r != ROUTER_ADDED_SUCCESSFULLY)
+    extrainfo_free(ei);
+
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+  return r;
+}
+
+#define should_cache_old_descriptors() \
+  directory_caches_dir_info(get_options())
+
+/** If we're a directory cache and routerlist <b>rl</b> doesn't have
+ * a copy of router <b>ri</b> yet, add it to the list of old (not
+ * recommended but still served) descriptors. Else free it. */
+static void
+routerlist_insert_old(routerlist_t *rl, routerinfo_t *ri)
+{
+  {
+    const routerinfo_t *ri_generated = router_get_my_routerinfo();
+    tor_assert(ri_generated != ri);
+  }
+  tor_assert(ri->cache_info.routerlist_index == -1);
+
+  if (should_cache_old_descriptors() &&
+      ri->purpose == ROUTER_PURPOSE_GENERAL &&
+      !sdmap_get(rl->desc_digest_map,
+                 ri->cache_info.signed_descriptor_digest)) {
+    signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri);
+    sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd);
+    smartlist_add(rl->old_routers, sd);
+    sd->routerlist_index = smartlist_len(rl->old_routers)-1;
+    if (!tor_digest_is_zero(sd->extra_info_digest))
+      sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd);
+  } else {
+    routerinfo_free(ri);
+  }
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Remove an item <b>ri</b> from the routerlist <b>rl</b>, updating indices
+ * as needed. If <b>idx</b> is nonnegative and smartlist_get(rl-&gt;routers,
+ * idx) == ri, we don't need to do a linear search over the list to decide
+ * which to remove.  We fill the gap in rl-&gt;routers with a later element in
+ * the list, if any exists. <b>ri</b> is freed.
+ *
+ * If <b>make_old</b> is true, instead of deleting the router, we try adding
+ * it to rl-&gt;old_routers. */
+void
+routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, time_t now)
+{
+  routerinfo_t *ri_tmp;
+  extrainfo_t *ei_tmp;
+  int idx = ri->cache_info.routerlist_index;
+  tor_assert(0 <= idx && idx < smartlist_len(rl->routers));
+  tor_assert(smartlist_get(rl->routers, idx) == ri);
+
+  nodelist_remove_routerinfo(ri);
+
+  /* make sure the rephist module knows that it's not running */
+  rep_hist_note_router_unreachable(ri->cache_info.identity_digest, now);
+
+  ri->cache_info.routerlist_index = -1;
+  smartlist_del(rl->routers, idx);
+  if (idx < smartlist_len(rl->routers)) {
+    routerinfo_t *r = smartlist_get(rl->routers, idx);
+    r->cache_info.routerlist_index = idx;
+  }
+
+  ri_tmp = rimap_remove(rl->identity_map, ri->cache_info.identity_digest);
+  router_dir_info_changed();
+  tor_assert(ri_tmp == ri);
+
+  if (make_old && should_cache_old_descriptors() &&
+      ri->purpose == ROUTER_PURPOSE_GENERAL) {
+    signed_descriptor_t *sd;
+    sd = signed_descriptor_from_routerinfo(ri);
+    smartlist_add(rl->old_routers, sd);
+    sd->routerlist_index = smartlist_len(rl->old_routers)-1;
+    sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd);
+    if (!tor_digest_is_zero(sd->extra_info_digest))
+      sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd);
+  } else {
+    signed_descriptor_t *sd_tmp;
+    sd_tmp = sdmap_remove(rl->desc_digest_map,
+                          ri->cache_info.signed_descriptor_digest);
+    tor_assert(sd_tmp == &(ri->cache_info));
+    rl->desc_store.bytes_dropped += ri->cache_info.signed_descriptor_len;
+    ei_tmp = eimap_remove(rl->extra_info_map,
+                          ri->cache_info.extra_info_digest);
+    if (ei_tmp) {
+      rl->extrainfo_store.bytes_dropped +=
+        ei_tmp->cache_info.signed_descriptor_len;
+      extrainfo_free(ei_tmp);
+    }
+    if (!tor_digest_is_zero(ri->cache_info.extra_info_digest))
+      sdmap_remove(rl->desc_by_eid_map, ri->cache_info.extra_info_digest);
+    routerinfo_free(ri);
+  }
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Remove a signed_descriptor_t <b>sd</b> from <b>rl</b>-\>old_routers, and
+ * adjust <b>rl</b> as appropriate.  <b>idx</b> is -1, or the index of
+ * <b>sd</b>. */
+static void
+routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx)
+{
+  signed_descriptor_t *sd_tmp;
+  extrainfo_t *ei_tmp;
+  desc_store_t *store;
+  if (idx == -1) {
+    idx = sd->routerlist_index;
+  }
+  tor_assert(0 <= idx && idx < smartlist_len(rl->old_routers));
+  /* XXXX edmanm's bridge relay triggered the following assert while
+   * running 0.2.0.12-alpha.  If anybody triggers this again, see if we
+   * can get a backtrace. */
+  tor_assert(smartlist_get(rl->old_routers, idx) == sd);
+  tor_assert(idx == sd->routerlist_index);
+
+  sd->routerlist_index = -1;
+  smartlist_del(rl->old_routers, idx);
+  if (idx < smartlist_len(rl->old_routers)) {
+    signed_descriptor_t *d = smartlist_get(rl->old_routers, idx);
+    d->routerlist_index = idx;
+  }
+  sd_tmp = sdmap_remove(rl->desc_digest_map,
+                        sd->signed_descriptor_digest);
+  tor_assert(sd_tmp == sd);
+  store = desc_get_store(rl, sd);
+  if (store)
+    store->bytes_dropped += sd->signed_descriptor_len;
+
+  ei_tmp = eimap_remove(rl->extra_info_map,
+                        sd->extra_info_digest);
+  if (ei_tmp) {
+    rl->extrainfo_store.bytes_dropped +=
+      ei_tmp->cache_info.signed_descriptor_len;
+    extrainfo_free(ei_tmp);
+  }
+  if (!tor_digest_is_zero(sd->extra_info_digest))
+    sdmap_remove(rl->desc_by_eid_map, sd->extra_info_digest);
+
+  signed_descriptor_free(sd);
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Remove <b>ri_old</b> from the routerlist <b>rl</b>, and replace it with
+ * <b>ri_new</b>, updating all index info.  If <b>idx</b> is nonnegative and
+ * smartlist_get(rl-&gt;routers, idx) == ri, we don't need to do a linear
+ * search over the list to decide which to remove.  We put ri_new in the same
+ * index as ri_old, if possible.  ri is freed as appropriate.
+ *
+ * If should_cache_descriptors() is true, instead of deleting the router,
+ * we add it to rl-&gt;old_routers. */
+static void
+routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old,
+                   routerinfo_t *ri_new)
+{
+  int idx;
+  int same_descriptors;
+
+  routerinfo_t *ri_tmp;
+  extrainfo_t *ei_tmp;
+  {
+    const routerinfo_t *ri_generated = router_get_my_routerinfo();
+    tor_assert(ri_generated != ri_new);
+  }
+  tor_assert(ri_old != ri_new);
+  tor_assert(ri_new->cache_info.routerlist_index == -1);
+
+  idx = ri_old->cache_info.routerlist_index;
+  tor_assert(0 <= idx && idx < smartlist_len(rl->routers));
+  tor_assert(smartlist_get(rl->routers, idx) == ri_old);
+
+  {
+    routerinfo_t *ri_old_tmp=NULL;
+    nodelist_set_routerinfo(ri_new, &ri_old_tmp);
+    tor_assert(ri_old == ri_old_tmp);
+  }
+
+  router_dir_info_changed();
+  if (idx >= 0) {
+    smartlist_set(rl->routers, idx, ri_new);
+    ri_old->cache_info.routerlist_index = -1;
+    ri_new->cache_info.routerlist_index = idx;
+    /* Check that ri_old is not in rl->routers anymore: */
+    tor_assert( routerlist_find_elt_(rl->routers, ri_old, -1) == -1 );
+  } else {
+    log_warn(LD_BUG, "Appending entry from routerlist_replace.");
+    routerlist_insert(rl, ri_new);
+    return;
+  }
+  if (tor_memneq(ri_old->cache_info.identity_digest,
+             ri_new->cache_info.identity_digest, DIGEST_LEN)) {
+    /* digests don't match; digestmap_set won't replace */
+    rimap_remove(rl->identity_map, ri_old->cache_info.identity_digest);
+  }
+  ri_tmp = rimap_set(rl->identity_map,
+                     ri_new->cache_info.identity_digest, ri_new);
+  tor_assert(!ri_tmp || ri_tmp == ri_old);
+  sdmap_set(rl->desc_digest_map,
+            ri_new->cache_info.signed_descriptor_digest,
+            &(ri_new->cache_info));
+
+  if (!tor_digest_is_zero(ri_new->cache_info.extra_info_digest)) {
+    sdmap_set(rl->desc_by_eid_map, ri_new->cache_info.extra_info_digest,
+              &ri_new->cache_info);
+  }
+
+  same_descriptors = tor_memeq(ri_old->cache_info.signed_descriptor_digest,
+                              ri_new->cache_info.signed_descriptor_digest,
+                              DIGEST_LEN);
+
+  if (should_cache_old_descriptors() &&
+      ri_old->purpose == ROUTER_PURPOSE_GENERAL &&
+      !same_descriptors) {
+    /* ri_old is going to become a signed_descriptor_t and go into
+     * old_routers */
+    signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri_old);
+    smartlist_add(rl->old_routers, sd);
+    sd->routerlist_index = smartlist_len(rl->old_routers)-1;
+    sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd);
+    if (!tor_digest_is_zero(sd->extra_info_digest))
+      sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd);
+  } else {
+    /* We're dropping ri_old. */
+    if (!same_descriptors) {
+      /* digests don't match; The sdmap_set above didn't replace */
+      sdmap_remove(rl->desc_digest_map,
+                   ri_old->cache_info.signed_descriptor_digest);
+
+      if (tor_memneq(ri_old->cache_info.extra_info_digest,
+                 ri_new->cache_info.extra_info_digest, DIGEST_LEN)) {
+        ei_tmp = eimap_remove(rl->extra_info_map,
+                              ri_old->cache_info.extra_info_digest);
+        if (ei_tmp) {
+          rl->extrainfo_store.bytes_dropped +=
+            ei_tmp->cache_info.signed_descriptor_len;
+          extrainfo_free(ei_tmp);
+        }
+      }
+
+      if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) {
+        sdmap_remove(rl->desc_by_eid_map,
+                     ri_old->cache_info.extra_info_digest);
+      }
+    }
+    rl->desc_store.bytes_dropped += ri_old->cache_info.signed_descriptor_len;
+    routerinfo_free(ri_old);
+  }
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Extract the descriptor <b>sd</b> from old_routerlist, and re-parse
+ * it as a fresh routerinfo_t. */
+static routerinfo_t *
+routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd)
+{
+  routerinfo_t *ri;
+  const char *body;
+
+  body = signed_descriptor_get_annotations(sd);
+
+  ri = router_parse_entry_from_string(body,
+                         body+sd->signed_descriptor_len+sd->annotations_len,
+                         0, 1, NULL, NULL);
+  if (!ri)
+    return NULL;
+  signed_descriptor_move(&ri->cache_info, sd);
+
+  routerlist_remove_old(rl, sd, -1);
+
+  return ri;
+}
+
+/** Free all memory held by the routerlist module.
+ * Note: Calling routerlist_free_all() should always be paired with
+ * a call to nodelist_free_all(). These should only be called during
+ * cleanup.
+ */
+void
+routerlist_free_all(void)
+{
+  routerlist_free(routerlist);
+  routerlist = NULL;
+  if (warned_nicknames) {
+    SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp));
+    smartlist_free(warned_nicknames);
+    warned_nicknames = NULL;
+  }
+  clear_dir_servers();
+  smartlist_free(trusted_dir_servers);
+  smartlist_free(fallback_dir_servers);
+  trusted_dir_servers = fallback_dir_servers = NULL;
+  if (trusted_dir_certs) {
+    digestmap_free(trusted_dir_certs, cert_list_free_void);
+    trusted_dir_certs = NULL;
+  }
+}
+
+/** Forget that we have issued any router-related warnings, so that we'll
+ * warn again if we see the same errors. */
+void
+routerlist_reset_warnings(void)
+{
+  if (!warned_nicknames)
+    warned_nicknames = smartlist_new();
+  SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp));
+  smartlist_clear(warned_nicknames); /* now the list is empty. */
+
+  networkstatus_reset_warnings();
+}
+
+/** Return 1 if the signed descriptor of this router is older than
+ *  <b>seconds</b> seconds.  Otherwise return 0. */
+MOCK_IMPL(int,
+router_descriptor_is_older_than,(const routerinfo_t *router, int seconds))
+{
+  return router->cache_info.published_on < approx_time() - seconds;
+}
+
+/** Add <b>router</b> to the routerlist, if we don't already have it.  Replace
+ * older entries (if any) with the same key.  Note: Callers should not hold
+ * their pointers to <b>router</b> if this function fails; <b>router</b>
+ * will either be inserted into the routerlist or freed. Similarly, even
+ * if this call succeeds, they should not hold their pointers to
+ * <b>router</b> after subsequent calls with other routerinfo's -- they
+ * might cause the original routerinfo to get freed.
+ *
+ * Returns the status for the operation. Might set *<b>msg</b> if it wants
+ * the poster of the router to know something.
+ *
+ * If <b>from_cache</b>, this descriptor came from our disk cache. If
+ * <b>from_fetch</b>, we received it in response to a request we made.
+ * (If both are false, that means it was uploaded to us as an auth dir
+ * server or via the controller.)
+ *
+ * This function should be called *after*
+ * routers_update_status_from_consensus_networkstatus; subsequently, you
+ * should call router_rebuild_store and routerlist_descriptors_added.
+ */
+was_router_added_t
+router_add_to_routerlist(routerinfo_t *router, const char **msg,
+                         int from_cache, int from_fetch)
+{
+  const char *id_digest;
+  const or_options_t *options = get_options();
+  int authdir = authdir_mode_handles_descs(options, router->purpose);
+  int authdir_believes_valid = 0;
+  routerinfo_t *old_router;
+  networkstatus_t *consensus =
+    networkstatus_get_latest_consensus_by_flavor(FLAV_NS);
+  int in_consensus = 0;
+
+  tor_assert(msg);
+
+  if (!routerlist)
+    router_get_routerlist();
+
+  id_digest = router->cache_info.identity_digest;
+
+  old_router = router_get_mutable_by_digest(id_digest);
+
+  /* Make sure that it isn't expired. */
+  if (router->cert_expiration_time < approx_time()) {
+    routerinfo_free(router);
+    *msg = "Some certs on this router are expired.";
+    return ROUTER_CERTS_EXPIRED;
+  }
+
+  /* Make sure that we haven't already got this exact descriptor. */
+  if (sdmap_get(routerlist->desc_digest_map,
+                router->cache_info.signed_descriptor_digest)) {
+    /* If we have this descriptor already and the new descriptor is a bridge
+     * descriptor, replace it. If we had a bridge descriptor before and the
+     * new one is not a bridge descriptor, don't replace it. */
+
+    /* Only members of routerlist->identity_map can be bridges; we don't
+     * put bridges in old_routers. */
+    const int was_bridge = old_router &&
+      old_router->purpose == ROUTER_PURPOSE_BRIDGE;
+
+    if (routerinfo_is_a_configured_bridge(router) &&
+        router->purpose == ROUTER_PURPOSE_BRIDGE &&
+        !was_bridge) {
+      log_info(LD_DIR, "Replacing non-bridge descriptor with bridge "
+               "descriptor for router %s",
+               router_describe(router));
+    } else {
+      log_info(LD_DIR,
+               "Dropping descriptor that we already have for router %s",
+               router_describe(router));
+      *msg = "Router descriptor was not new.";
+      routerinfo_free(router);
+      return ROUTER_IS_ALREADY_KNOWN;
+    }
+  }
+
+  if (authdir) {
+    if (authdir_wants_to_reject_router(router, msg,
+                                       !from_cache && !from_fetch,
+                                       &authdir_believes_valid)) {
+      tor_assert(*msg);
+      routerinfo_free(router);
+      return ROUTER_AUTHDIR_REJECTS;
+    }
+  } else if (from_fetch) {
+    /* Only check the descriptor digest against the network statuses when
+     * we are receiving in response to a fetch. */
+
+    if (!signed_desc_digest_is_recognized(&router->cache_info) &&
+        !routerinfo_is_a_configured_bridge(router)) {
+      /* We asked for it, so some networkstatus must have listed it when we
+       * did.  Save it if we're a cache in case somebody else asks for it. */
+      log_info(LD_DIR,
+               "Received a no-longer-recognized descriptor for router %s",
+               router_describe(router));
+      *msg = "Router descriptor is not referenced by any network-status.";
+
+      /* Only journal this desc if we want to keep old descriptors */
+      if (!from_cache && should_cache_old_descriptors())
+        signed_desc_append_to_journal(&router->cache_info,
+                                      &routerlist->desc_store);
+      routerlist_insert_old(routerlist, router);
+      return ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS;
+    }
+  }
+
+  /* We no longer need a router with this descriptor digest. */
+  if (consensus) {
+    routerstatus_t *rs = networkstatus_vote_find_mutable_entry(
+                                                     consensus, id_digest);
+    if (rs && tor_memeq(rs->descriptor_digest,
+                      router->cache_info.signed_descriptor_digest,
+                      DIGEST_LEN)) {
+      in_consensus = 1;
+    }
+  }
+
+  if (router->purpose == ROUTER_PURPOSE_GENERAL &&
+      consensus && !in_consensus && !authdir) {
+    /* If it's a general router not listed in the consensus, then don't
+     * consider replacing the latest router with it. */
+    if (!from_cache && should_cache_old_descriptors())
+      signed_desc_append_to_journal(&router->cache_info,
+                                    &routerlist->desc_store);
+    routerlist_insert_old(routerlist, router);
+    *msg = "Skipping router descriptor: not in consensus.";
+    return ROUTER_NOT_IN_CONSENSUS;
+  }
+
+  /* If we're reading a bridge descriptor from our cache, and we don't
+   * recognize it as one of our currently configured bridges, drop the
+   * descriptor. Otherwise we could end up using it as one of our entry
+   * guards even if it isn't in our Bridge config lines. */
+  if (router->purpose == ROUTER_PURPOSE_BRIDGE && from_cache &&
+      !authdir_mode_bridge(options) &&
+      !routerinfo_is_a_configured_bridge(router)) {
+    log_info(LD_DIR, "Dropping bridge descriptor for %s because we have "
+             "no bridge configured at that address.",
+             safe_str_client(router_describe(router)));
+    *msg = "Router descriptor was not a configured bridge.";
+    routerinfo_free(router);
+    return ROUTER_WAS_NOT_WANTED;
+  }
+
+  /* If we have a router with the same identity key, choose the newer one. */
+  if (old_router) {
+    if (!in_consensus && (router->cache_info.published_on <=
+                          old_router->cache_info.published_on)) {
+      /* Same key, but old.  This one is not listed in the consensus. */
+      log_debug(LD_DIR, "Not-new descriptor for router %s",
+                router_describe(router));
+      /* Only journal this desc if we'll be serving it. */
+      if (!from_cache && should_cache_old_descriptors())
+        signed_desc_append_to_journal(&router->cache_info,
+                                      &routerlist->desc_store);
+      routerlist_insert_old(routerlist, router);
+      *msg = "Router descriptor was not new.";
+      return ROUTER_IS_ALREADY_KNOWN;
+    } else {
+      /* Same key, and either new, or listed in the consensus. */
+      log_debug(LD_DIR, "Replacing entry for router %s",
+                router_describe(router));
+      routerlist_replace(routerlist, old_router, router);
+      if (!from_cache) {
+        signed_desc_append_to_journal(&router->cache_info,
+                                      &routerlist->desc_store);
+      }
+      *msg = authdir_believes_valid ? "Valid server updated" :
+        ("Invalid server updated. (This dirserver is marking your "
+         "server as unapproved.)");
+      return ROUTER_ADDED_SUCCESSFULLY;
+    }
+  }
+
+  if (!in_consensus && from_cache &&
+      router_descriptor_is_older_than(router, OLD_ROUTER_DESC_MAX_AGE)) {
+    *msg = "Router descriptor was really old.";
+    routerinfo_free(router);
+    return ROUTER_WAS_TOO_OLD;
+  }
+
+  /* We haven't seen a router with this identity before. Add it to the end of
+   * the list. */
+  routerlist_insert(routerlist, router);
+  if (!from_cache) {
+    signed_desc_append_to_journal(&router->cache_info,
+                                  &routerlist->desc_store);
+  }
+  return ROUTER_ADDED_SUCCESSFULLY;
+}
+
+/** Insert <b>ei</b> into the routerlist, or free it. Other arguments are
+ * as for router_add_to_routerlist().  Return ROUTER_ADDED_SUCCESSFULLY iff
+ * we actually inserted it, ROUTER_BAD_EI otherwise.
+ */
+was_router_added_t
+router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg,
+                                   int from_cache, int from_fetch)
+{
+  was_router_added_t inserted;
+  (void)from_fetch;
+  if (msg) *msg = NULL;
+  /*XXXX Do something with msg */
+
+  inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache);
+
+  if (WRA_WAS_ADDED(inserted) && !from_cache)
+    signed_desc_append_to_journal(&ei->cache_info,
+                                  &routerlist->extrainfo_store);
+
+  return inserted;
+}
+
+/** Sorting helper: return &lt;0, 0, or &gt;0 depending on whether the
+ * signed_descriptor_t* in *<b>a</b> has an identity digest preceding, equal
+ * to, or later than that of *<b>b</b>. */
+static int
+compare_old_routers_by_identity_(const void **_a, const void **_b)
+{
+  int i;
+  const signed_descriptor_t *r1 = *_a, *r2 = *_b;
+  if ((i = fast_memcmp(r1->identity_digest, r2->identity_digest, DIGEST_LEN)))
+    return i;
+  return (int)(r1->published_on - r2->published_on);
+}
+
+/** Internal type used to represent how long an old descriptor was valid,
+ * where it appeared in the list of old descriptors, and whether it's extra
+ * old. Used only by routerlist_remove_old_cached_routers_with_id(). */
+struct duration_idx_t {
+  int duration;
+  int idx;
+  int old;
+};
+
+/** Sorting helper: compare two duration_idx_t by their duration. */
+static int
+compare_duration_idx_(const void *_d1, const void *_d2)
+{
+  const struct duration_idx_t *d1 = _d1;
+  const struct duration_idx_t *d2 = _d2;
+  return d1->duration - d2->duration;
+}
+
+/** The range <b>lo</b> through <b>hi</b> inclusive of routerlist->old_routers
+ * must contain routerinfo_t with the same identity and with publication time
+ * in ascending order.  Remove members from this range until there are no more
+ * than max_descriptors_per_router() remaining.  Start by removing the oldest
+ * members from before <b>cutoff</b>, then remove members which were current
+ * for the lowest amount of time.  The order of members of old_routers at
+ * indices <b>lo</b> or higher may be changed.
+ */
+static void
+routerlist_remove_old_cached_routers_with_id(time_t now,
+                                             time_t cutoff, int lo, int hi,
+                                             digestset_t *retain)
+{
+  int i, n = hi-lo+1;
+  unsigned n_extra, n_rmv = 0;
+  struct duration_idx_t *lifespans;
+  uint8_t *rmv, *must_keep;
+  smartlist_t *lst = routerlist->old_routers;
+#if 1
+  const char *ident;
+  tor_assert(hi < smartlist_len(lst));
+  tor_assert(lo <= hi);
+  ident = ((signed_descriptor_t*)smartlist_get(lst, lo))->identity_digest;
+  for (i = lo+1; i <= hi; ++i) {
+    signed_descriptor_t *r = smartlist_get(lst, i);
+    tor_assert(tor_memeq(ident, r->identity_digest, DIGEST_LEN));
+  }
+#endif /* 1 */
+  /* Check whether we need to do anything at all. */
+  {
+    int mdpr = directory_caches_dir_info(get_options()) ? 2 : 1;
+    if (n <= mdpr)
+      return;
+    n_extra = n - mdpr;
+  }
+
+  lifespans = tor_calloc(n, sizeof(struct duration_idx_t));
+  rmv = tor_calloc(n, sizeof(uint8_t));
+  must_keep = tor_calloc(n, sizeof(uint8_t));
+  /* Set lifespans to contain the lifespan and index of each server. */
+  /* Set rmv[i-lo]=1 if we're going to remove a server for being too old. */
+  for (i = lo; i <= hi; ++i) {
+    signed_descriptor_t *r = smartlist_get(lst, i);
+    signed_descriptor_t *r_next;
+    lifespans[i-lo].idx = i;
+    if (r->last_listed_as_valid_until >= now ||
+        (retain && digestset_probably_contains(retain,
+                                               r->signed_descriptor_digest))) {
+      must_keep[i-lo] = 1;
+    }
+    if (i < hi) {
+      r_next = smartlist_get(lst, i+1);
+      tor_assert(r->published_on <= r_next->published_on);
+      lifespans[i-lo].duration = (int)(r_next->published_on - r->published_on);
+    } else {
+      r_next = NULL;
+      lifespans[i-lo].duration = INT_MAX;
+    }
+    if (!must_keep[i-lo] && r->published_on < cutoff && n_rmv < n_extra) {
+      ++n_rmv;
+      lifespans[i-lo].old = 1;
+      rmv[i-lo] = 1;
+    }
+  }
+
+  if (n_rmv < n_extra) {
+    /**
+     * We aren't removing enough servers for being old.  Sort lifespans by
+     * the duration of liveness, and remove the ones we're not already going to
+     * remove based on how long they were alive.
+     **/
+    qsort(lifespans, n, sizeof(struct duration_idx_t), compare_duration_idx_);
+    for (i = 0; i < n && n_rmv < n_extra; ++i) {
+      if (!must_keep[lifespans[i].idx-lo] && !lifespans[i].old) {
+        rmv[lifespans[i].idx-lo] = 1;
+        ++n_rmv;
+      }
+    }
+  }
+
+  i = hi;
+  do {
+    if (rmv[i-lo])
+      routerlist_remove_old(routerlist, smartlist_get(lst, i), i);
+  } while (--i >= lo);
+  tor_free(must_keep);
+  tor_free(rmv);
+  tor_free(lifespans);
+}
+
+/** Deactivate any routers from the routerlist that are more than
+ * ROUTER_MAX_AGE seconds old and not recommended by any networkstatuses;
+ * remove old routers from the list of cached routers if we have too many.
+ */
+void
+routerlist_remove_old_routers(void)
+{
+  int i, hi=-1;
+  const char *cur_id = NULL;
+  time_t now = time(NULL);
+  time_t cutoff;
+  routerinfo_t *router;
+  signed_descriptor_t *sd;
+  digestset_t *retain;
+  const networkstatus_t *consensus = networkstatus_get_latest_consensus();
+
+  trusted_dirs_remove_old_certs();
+
+  if (!routerlist || !consensus)
+    return;
+
+  // routerlist_assert_ok(routerlist);
+
+  /* We need to guess how many router descriptors we will wind up wanting to
+     retain, so that we can be sure to allocate a large enough Bloom filter
+     to hold the digest set.  Overestimating is fine; underestimating is bad.
+  */
+  {
+    /* We'll probably retain everything in the consensus. */
+    int n_max_retain = smartlist_len(consensus->routerstatus_list);
+    retain = digestset_new(n_max_retain);
+  }
+
+  cutoff = now - OLD_ROUTER_DESC_MAX_AGE;
+  /* Retain anything listed in the consensus. */
+  if (consensus) {
+    SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs,
+        if (rs->published_on >= cutoff)
+          digestset_add(retain, rs->descriptor_digest));
+  }
+
+  /* If we have a consensus, we should consider pruning current routers that
+   * are too old and that nobody recommends.  (If we don't have a consensus,
+   * then we should get one before we decide to kill routers.) */
+
+  if (consensus) {
+    cutoff = now - ROUTER_MAX_AGE;
+    /* Remove too-old unrecommended members of routerlist->routers. */
+    for (i = 0; i < smartlist_len(routerlist->routers); ++i) {
+      router = smartlist_get(routerlist->routers, i);
+      if (router->cache_info.published_on <= cutoff &&
+          router->cache_info.last_listed_as_valid_until < now &&
+          !digestset_probably_contains(retain,
+                          router->cache_info.signed_descriptor_digest)) {
+        /* Too old: remove it.  (If we're a cache, just move it into
+         * old_routers.) */
+        log_info(LD_DIR,
+                 "Forgetting obsolete (too old) routerinfo for router %s",
+                 router_describe(router));
+        routerlist_remove(routerlist, router, 1, now);
+        i--;
+      }
+    }
+  }
+
+  //routerlist_assert_ok(routerlist);
+
+  /* Remove far-too-old members of routerlist->old_routers. */
+  cutoff = now - OLD_ROUTER_DESC_MAX_AGE;
+  for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) {
+    sd = smartlist_get(routerlist->old_routers, i);
+    if (sd->published_on <= cutoff &&
+        sd->last_listed_as_valid_until < now &&
+        !digestset_probably_contains(retain, sd->signed_descriptor_digest)) {
+      /* Too old.  Remove it. */
+      routerlist_remove_old(routerlist, sd, i--);
+    }
+  }
+
+  //routerlist_assert_ok(routerlist);
+
+  log_info(LD_DIR, "We have %d live routers and %d old router descriptors.",
+           smartlist_len(routerlist->routers),
+           smartlist_len(routerlist->old_routers));
+
+  /* Now we might have to look at routerlist->old_routers for extraneous
+   * members. (We'd keep all the members if we could, but we need to save
+   * space.) First, check whether we have too many router descriptors, total.
+   * We're okay with having too many for some given router, so long as the
+   * total number doesn't approach max_descriptors_per_router()*len(router).
+   */
+  if (smartlist_len(routerlist->old_routers) <
+      smartlist_len(routerlist->routers))
+    goto done;
+
+  /* Sort by identity, then fix indices. */
+  smartlist_sort(routerlist->old_routers, compare_old_routers_by_identity_);
+  /* Fix indices. */
+  for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) {
+    signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i);
+    r->routerlist_index = i;
+  }
+
+  /* Iterate through the list from back to front, so when we remove descriptors
+   * we don't mess up groups we haven't gotten to. */
+  for (i = smartlist_len(routerlist->old_routers)-1; i >= 0; --i) {
+    signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i);
+    if (!cur_id) {
+      cur_id = r->identity_digest;
+      hi = i;
+    }
+    if (tor_memneq(cur_id, r->identity_digest, DIGEST_LEN)) {
+      routerlist_remove_old_cached_routers_with_id(now,
+                                                   cutoff, i+1, hi, retain);
+      cur_id = r->identity_digest;
+      hi = i;
+    }
+  }
+  if (hi>=0)
+    routerlist_remove_old_cached_routers_with_id(now, cutoff, 0, hi, retain);
+  //routerlist_assert_ok(routerlist);
+
+ done:
+  digestset_free(retain);
+  router_rebuild_store(RRS_DONT_REMOVE_OLD, &routerlist->desc_store);
+  router_rebuild_store(RRS_DONT_REMOVE_OLD,&routerlist->extrainfo_store);
+}
+
+/** We just added a new set of descriptors. Take whatever extra steps
+ * we need. */
+void
+routerlist_descriptors_added(smartlist_t *sl, int from_cache)
+{
+  tor_assert(sl);
+  control_event_descriptors_changed(sl);
+  SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) {
+    if (ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      learned_bridge_descriptor(ri, from_cache);
+    if (ri->needs_retest_if_added) {
+      ri->needs_retest_if_added = 0;
+      dirserv_single_reachability_test(approx_time(), ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/**
+ * Code to parse a single router descriptor and insert it into the
+ * routerlist.  Return -1 if the descriptor was ill-formed; 0 if the
+ * descriptor was well-formed but could not be added; and 1 if the
+ * descriptor was added.
+ *
+ * If we don't add it and <b>msg</b> is not NULL, then assign to
+ * *<b>msg</b> a static string describing the reason for refusing the
+ * descriptor.
+ *
+ * This is used only by the controller.
+ */
+int
+router_load_single_router(const char *s, uint8_t purpose, int cache,
+                          const char **msg)
+{
+  routerinfo_t *ri;
+  was_router_added_t r;
+  smartlist_t *lst;
+  char annotation_buf[ROUTER_ANNOTATION_BUF_LEN];
+  tor_assert(msg);
+  *msg = NULL;
+
+  tor_snprintf(annotation_buf, sizeof(annotation_buf),
+               "@source controller\n"
+               "@purpose %s\n", router_purpose_to_string(purpose));
+
+  if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0,
+                                            annotation_buf, NULL))) {
+    log_warn(LD_DIR, "Error parsing router descriptor; dropping.");
+    *msg = "Couldn't parse router descriptor.";
+    return -1;
+  }
+  tor_assert(ri->purpose == purpose);
+  if (router_is_me(ri)) {
+    log_warn(LD_DIR, "Router's identity key matches mine; dropping.");
+    *msg = "Router's identity key matches mine.";
+    routerinfo_free(ri);
+    return 0;
+  }
+
+  if (!cache) /* obey the preference of the controller */
+    ri->cache_info.do_not_cache = 1;
+
+  lst = smartlist_new();
+  smartlist_add(lst, ri);
+  routers_update_status_from_consensus_networkstatus(lst, 0);
+
+  r = router_add_to_routerlist(ri, msg, 0, 0);
+  if (!WRA_WAS_ADDED(r)) {
+    /* we've already assigned to *msg now, and ri is already freed */
+    tor_assert(*msg);
+    if (r == ROUTER_AUTHDIR_REJECTS)
+      log_warn(LD_DIR, "Couldn't add router to list: %s Dropping.", *msg);
+    smartlist_free(lst);
+    return 0;
+  } else {
+    routerlist_descriptors_added(lst, 0);
+    smartlist_free(lst);
+    log_debug(LD_DIR, "Added router to list");
+    return 1;
+  }
+}
+
+/** Given a string <b>s</b> containing some routerdescs, parse it and put the
+ * routers into our directory.  If saved_location is SAVED_NOWHERE, the routers
+ * are in response to a query to the network: cache them by adding them to
+ * the journal.
+ *
+ * Return the number of routers actually added.
+ *
+ * If <b>requested_fingerprints</b> is provided, it must contain a list of
+ * uppercased fingerprints.  Do not update any router whose
+ * fingerprint is not on the list; after updating a router, remove its
+ * fingerprint from the list.
+ *
+ * If <b>descriptor_digests</b> is non-zero, then the requested_fingerprints
+ * are descriptor digests. Otherwise they are identity digests.
+ */
+int
+router_load_routers_from_string(const char *s, const char *eos,
+                                saved_location_t saved_location,
+                                smartlist_t *requested_fingerprints,
+                                int descriptor_digests,
+                                const char *prepend_annotations)
+{
+  smartlist_t *routers = smartlist_new(), *changed = smartlist_new();
+  char fp[HEX_DIGEST_LEN+1];
+  const char *msg;
+  int from_cache = (saved_location != SAVED_NOWHERE);
+  int allow_annotations = (saved_location != SAVED_NOWHERE);
+  int any_changed = 0;
+  smartlist_t *invalid_digests = smartlist_new();
+
+  router_parse_list_from_string(&s, eos, routers, saved_location, 0,
+                                allow_annotations, prepend_annotations,
+                                invalid_digests);
+
+  routers_update_status_from_consensus_networkstatus(routers, !from_cache);
+
+  log_info(LD_DIR, "%d elements to add", smartlist_len(routers));
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    was_router_added_t r;
+    char d[DIGEST_LEN];
+    if (requested_fingerprints) {
+      base16_encode(fp, sizeof(fp), descriptor_digests ?
+                      ri->cache_info.signed_descriptor_digest :
+                      ri->cache_info.identity_digest,
+                    DIGEST_LEN);
+      if (smartlist_contains_string(requested_fingerprints, fp)) {
+        smartlist_string_remove(requested_fingerprints, fp);
+      } else {
+        char *requested =
+          smartlist_join_strings(requested_fingerprints," ",0,NULL);
+        log_warn(LD_DIR,
+                 "We received a router descriptor with a fingerprint (%s) "
+                 "that we never requested. (We asked for: %s.) Dropping.",
+                 fp, requested);
+        tor_free(requested);
+        routerinfo_free(ri);
+        continue;
+      }
+    }
+
+    memcpy(d, ri->cache_info.signed_descriptor_digest, DIGEST_LEN);
+    r = router_add_to_routerlist(ri, &msg, from_cache, !from_cache);
+    if (WRA_WAS_ADDED(r)) {
+      any_changed++;
+      smartlist_add(changed, ri);
+      routerlist_descriptors_added(changed, from_cache);
+      smartlist_clear(changed);
+    } else if (WRA_NEVER_DOWNLOADABLE(r)) {
+      download_status_t *dl_status;
+      dl_status = router_get_dl_status_by_descriptor_digest(d);
+      if (dl_status) {
+        log_info(LD_GENERAL, "Marking router %s as never downloadable",
+                 hex_str(d, DIGEST_LEN));
+        download_status_mark_impossible(dl_status);
+      }
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) {
+    /* This digest is never going to be parseable. */
+    base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN);
+    if (requested_fingerprints && descriptor_digests) {
+      if (! smartlist_contains_string(requested_fingerprints, fp)) {
+        /* But we didn't ask for it, so we should assume shennanegans. */
+        continue;
+      }
+      smartlist_string_remove(requested_fingerprints, fp);
+    }
+    download_status_t *dls;
+    dls = router_get_dl_status_by_descriptor_digest((char*)bad_digest);
+    if (dls) {
+      log_info(LD_GENERAL, "Marking router with descriptor %s as unparseable, "
+               "and therefore undownloadable", fp);
+      download_status_mark_impossible(dls);
+    }
+  } SMARTLIST_FOREACH_END(bad_digest);
+  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
+  smartlist_free(invalid_digests);
+
+  routerlist_assert_ok(routerlist);
+
+  if (any_changed)
+    router_rebuild_store(0, &routerlist->desc_store);
+
+  smartlist_free(routers);
+  smartlist_free(changed);
+
+  return any_changed;
+}
+
+/** Parse one or more extrainfos from <b>s</b> (ending immediately before
+ * <b>eos</b> if <b>eos</b> is present).  Other arguments are as for
+ * router_load_routers_from_string(). */
+void
+router_load_extrainfo_from_string(const char *s, const char *eos,
+                                  saved_location_t saved_location,
+                                  smartlist_t *requested_fingerprints,
+                                  int descriptor_digests)
+{
+  smartlist_t *extrainfo_list = smartlist_new();
+  const char *msg;
+  int from_cache = (saved_location != SAVED_NOWHERE);
+  smartlist_t *invalid_digests = smartlist_new();
+
+  router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0,
+                                NULL, invalid_digests);
+
+  log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list));
+
+  SMARTLIST_FOREACH_BEGIN(extrainfo_list, extrainfo_t *, ei) {
+      uint8_t d[DIGEST_LEN];
+      memcpy(d, ei->cache_info.signed_descriptor_digest, DIGEST_LEN);
+      was_router_added_t added =
+        router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache);
+      if (WRA_WAS_ADDED(added) && requested_fingerprints) {
+        char fp[HEX_DIGEST_LEN+1];
+        base16_encode(fp, sizeof(fp), descriptor_digests ?
+                        ei->cache_info.signed_descriptor_digest :
+                        ei->cache_info.identity_digest,
+                      DIGEST_LEN);
+        smartlist_string_remove(requested_fingerprints, fp);
+        /* We silently let relays stuff us with extrainfos we didn't ask for,
+         * so long as we would have wanted them anyway.  Since we always fetch
+         * all the extrainfos we want, and we never actually act on them
+         * inside Tor, this should be harmless. */
+      } else if (WRA_NEVER_DOWNLOADABLE(added)) {
+        signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d);
+        if (sd) {
+          log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
+                   "unparseable, and therefore undownloadable",
+                   hex_str((char*)d,DIGEST_LEN));
+          download_status_mark_impossible(&sd->ei_dl_status);
+        }
+      }
+  } SMARTLIST_FOREACH_END(ei);
+
+  SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) {
+    /* This digest is never going to be parseable. */
+    char fp[HEX_DIGEST_LEN+1];
+    base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN);
+    if (requested_fingerprints) {
+      if (! smartlist_contains_string(requested_fingerprints, fp)) {
+        /* But we didn't ask for it, so we should assume shennanegans. */
+        continue;
+      }
+      smartlist_string_remove(requested_fingerprints, fp);
+    }
+    signed_descriptor_t *sd =
+      router_get_by_extrainfo_digest((char*)bad_digest);
+    if (sd) {
+      log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
+               "unparseable, and therefore undownloadable", fp);
+      download_status_mark_impossible(&sd->ei_dl_status);
+    }
+  } SMARTLIST_FOREACH_END(bad_digest);
+  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
+  smartlist_free(invalid_digests);
+
+  routerlist_assert_ok(routerlist);
+  router_rebuild_store(0, &router_get_routerlist()->extrainfo_store);
+
+  smartlist_free(extrainfo_list);
+}
+
+/** Return true iff the latest ns-flavored consensus includes a descriptor
+ * whose digest is that of <b>desc</b>. */
+static int
+signed_desc_digest_is_recognized(signed_descriptor_t *desc)
+{
+  const routerstatus_t *rs;
+  networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(
+                                                                      FLAV_NS);
+
+  if (consensus) {
+    rs = networkstatus_vote_find_entry(consensus, desc->identity_digest);
+    if (rs && tor_memeq(rs->descriptor_digest,
+                      desc->signed_descriptor_digest, DIGEST_LEN))
+      return 1;
+  }
+  return 0;
+}
+
+/** Update downloads for router descriptors and/or microdescriptors as
+ * appropriate. */
+void
+update_all_descriptor_downloads(time_t now)
+{
+  if (should_delay_dir_fetches(get_options(), NULL))
+    return;
+  update_router_descriptor_downloads(now);
+  update_microdesc_downloads(now);
+  launch_dummy_descriptor_download_as_needed(now, get_options());
+}
+
+/** Clear all our timeouts for fetching v3 directory stuff, and then
+ * give it all a try again. */
+void
+routerlist_retry_directory_downloads(time_t now)
+{
+  (void)now;
+
+  log_debug(LD_GENERAL,
+            "In routerlist_retry_directory_downloads()");
+
+  router_reset_status_download_failures();
+  router_reset_descriptor_download_failures();
+  reschedule_directory_downloads();
+}
+
+/** Return true iff <b>router</b> does not permit exit streams.
+ */
+int
+router_exit_policy_rejects_all(const routerinfo_t *router)
+{
+  return router->policy_is_reject_star;
+}
+
+/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
+ * key <b>digest</b> which has DIGEST_LEN bytes.  If <b>address</b> is NULL,
+ * add ourself.  If <b>is_authority</b>, this is a directory authority.  Return
+ * the new directory server entry on success or NULL on failure. */
+static dir_server_t *
+dir_server_new(int is_authority,
+               const char *nickname,
+               const tor_addr_t *addr,
+               const char *hostname,
+               uint16_t dir_port, uint16_t or_port,
+               const tor_addr_port_t *addrport_ipv6,
+               const char *digest, const char *v3_auth_digest,
+               dirinfo_type_t type,
+               double weight)
+{
+  dir_server_t *ent;
+  uint32_t a;
+  char *hostname_ = NULL;
+
+  tor_assert(digest);
+
+  if (weight < 0)
+    return NULL;
+
+  if (tor_addr_family(addr) == AF_INET)
+    a = tor_addr_to_ipv4h(addr);
+  else
+    return NULL;
+
+  if (!hostname)
+    hostname_ = tor_addr_to_str_dup(addr);
+  else
+    hostname_ = tor_strdup(hostname);
+
+  ent = tor_malloc_zero(sizeof(dir_server_t));
+  ent->nickname = nickname ? tor_strdup(nickname) : NULL;
+  ent->address = hostname_;
+  ent->addr = a;
+  ent->dir_port = dir_port;
+  ent->or_port = or_port;
+  ent->is_running = 1;
+  ent->is_authority = is_authority;
+  ent->type = type;
+  ent->weight = weight;
+  if (addrport_ipv6) {
+    if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) {
+      log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6.");
+      tor_addr_make_unspec(&ent->ipv6_addr);
+    } else {
+      tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr);
+      ent->ipv6_orport = addrport_ipv6->port;
+    }
+  } else {
+    tor_addr_make_unspec(&ent->ipv6_addr);
+  }
+
+  memcpy(ent->digest, digest, DIGEST_LEN);
+  if (v3_auth_digest && (type & V3_DIRINFO))
+    memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
+
+  if (nickname)
+    tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d",
+                 nickname, hostname_, (int)dir_port);
+  else
+    tor_asprintf(&ent->description, "directory server at %s:%d",
+                 hostname_, (int)dir_port);
+
+  ent->fake_status.addr = ent->addr;
+  tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr);
+  memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(ent->fake_status.nickname, nickname,
+            sizeof(ent->fake_status.nickname));
+  else
+    ent->fake_status.nickname[0] = '\0';
+  ent->fake_status.dir_port = ent->dir_port;
+  ent->fake_status.or_port = ent->or_port;
+  ent->fake_status.ipv6_orport = ent->ipv6_orport;
+
+  return ent;
+}
+
+/** Create an authoritative directory server at
+ * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
+ * <b>address</b> is NULL, add ourself.  Return the new trusted directory
+ * server entry on success or NULL if we couldn't add it. */
+dir_server_t *
+trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *ipv6_addrport,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight)
+{
+  uint32_t a;
+  tor_addr_t addr;
+  char *hostname=NULL;
+  dir_server_t *result;
+
+  if (!address) { /* The address is us; we should guess. */
+    if (resolve_my_address(LOG_WARN, get_options(),
+                           &a, NULL, &hostname) < 0) {
+      log_warn(LD_CONFIG,
+               "Couldn't find a suitable address when adding ourself as a "
+               "trusted directory server.");
+      return NULL;
+    }
+    if (!hostname)
+      hostname = tor_dup_ip(a);
+  } else {
+    if (tor_lookup_hostname(address, &a)) {
+      log_warn(LD_CONFIG,
+               "Unable to lookup address for directory server at '%s'",
+               address);
+      return NULL;
+    }
+    hostname = tor_strdup(address);
+  }
+  tor_addr_from_ipv4h(&addr, a);
+
+  result = dir_server_new(1, nickname, &addr, hostname,
+                          dir_port, or_port,
+                          ipv6_addrport,
+                          digest,
+                          v3_auth_digest, type, weight);
+  tor_free(hostname);
+  return result;
+}
+
+/** Return a new dir_server_t for a fallback directory server at
+ * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
+ * <b>id_digest</b> */
+dir_server_t *
+fallback_dir_server_new(const tor_addr_t *addr,
+                        uint16_t dir_port, uint16_t or_port,
+                        const tor_addr_port_t *addrport_ipv6,
+                        const char *id_digest, double weight)
+{
+  return dir_server_new(0, NULL, addr, NULL, dir_port, or_port,
+                        addrport_ipv6,
+                        id_digest,
+                        NULL, ALL_DIRINFO, weight);
+}
+
+/** Add a directory server to the global list(s). */
+void
+dir_server_add(dir_server_t *ent)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  if (ent->is_authority)
+    smartlist_add(trusted_dir_servers, ent);
+
+  smartlist_add(fallback_dir_servers, ent);
+  router_dir_info_changed();
+}
+
+/** Free storage held in <b>cert</b>. */
+void
+authority_cert_free_(authority_cert_t *cert)
+{
+  if (!cert)
+    return;
+
+  tor_free(cert->cache_info.signed_descriptor_body);
+  crypto_pk_free(cert->signing_key);
+  crypto_pk_free(cert->identity_key);
+
+  tor_free(cert);
+}
+
+#define dir_server_free(val) \
+  FREE_AND_NULL(dir_server_t, dir_server_free_, (val))
+
+/** Free storage held in <b>ds</b>. */
+static void
+dir_server_free_(dir_server_t *ds)
+{
+  if (!ds)
+    return;
+
+  tor_free(ds->nickname);
+  tor_free(ds->description);
+  tor_free(ds->address);
+  tor_free(ds);
+}
+
+/** Remove all members from the list of dir servers. */
+void
+clear_dir_servers(void)
+{
+  if (fallback_dir_servers) {
+    SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
+                      dir_server_free(ent));
+    smartlist_clear(fallback_dir_servers);
+  } else {
+    fallback_dir_servers = smartlist_new();
+  }
+  if (trusted_dir_servers) {
+    smartlist_clear(trusted_dir_servers);
+  } else {
+    trusted_dir_servers = smartlist_new();
+  }
+  router_dir_info_changed();
+}
+
+/** For every current directory connection whose purpose is <b>purpose</b>,
+ * and where the resource being downloaded begins with <b>prefix</b>, split
+ * rest of the resource into base16 fingerprints (or base64 fingerprints if
+ * purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the
+ * corresponding elements of <b>result</b> to a nonzero value.
+ */
+static void
+list_pending_downloads(digestmap_t *result, digest256map_t *result256,
+                       int purpose, const char *prefix)
+{
+  const size_t p_len = strlen(prefix);
+  smartlist_t *tmp = smartlist_new();
+  smartlist_t *conns = get_connection_array();
+  int flags = DSR_HEX;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC)
+    flags = DSR_DIGEST256|DSR_BASE64;
+
+  tor_assert(result || result256);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_DIR &&
+        conn->purpose == purpose &&
+        !conn->marked_for_close) {
+      const char *resource = TO_DIR_CONN(conn)->requested_resource;
+      if (!strcmpstart(resource, prefix))
+        dir_split_resource_into_fingerprints(resource + p_len,
+                                             tmp, NULL, flags);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  if (result) {
+    SMARTLIST_FOREACH(tmp, char *, d,
+                    {
+                      digestmap_set(result, d, (void*)1);
+                      tor_free(d);
+                    });
+  } else if (result256) {
+    SMARTLIST_FOREACH(tmp, uint8_t *, d,
+                    {
+                      digest256map_set(result256, d, (void*)1);
+                      tor_free(d);
+                    });
+  }
+  smartlist_free(tmp);
+}
+
+/** For every router descriptor (or extra-info document if <b>extrainfo</b> is
+ * true) we are currently downloading by descriptor digest, set result[d] to
+ * (void*)1. */
+static void
+list_pending_descriptor_downloads(digestmap_t *result, int extrainfo)
+{
+  int purpose =
+    extrainfo ? DIR_PURPOSE_FETCH_EXTRAINFO : DIR_PURPOSE_FETCH_SERVERDESC;
+  list_pending_downloads(result, NULL, purpose, "d/");
+}
+
+/** For every microdescriptor we are currently downloading by descriptor
+ * digest, set result[d] to (void*)1.
+ */
+void
+list_pending_microdesc_downloads(digest256map_t *result)
+{
+  list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/");
+}
+
+/** For every certificate we are currently downloading by (identity digest,
+ * signing key digest) pair, set result[fp_pair] to (void *1).
+ */
+static void
+list_pending_fpsk_downloads(fp_pair_map_t *result)
+{
+  const char *pfx = "fp-sk/";
+  smartlist_t *tmp;
+  smartlist_t *conns;
+  const char *resource;
+
+  tor_assert(result);
+
+  tmp = smartlist_new();
+  conns = get_connection_array();
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_DIR &&
+        conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
+        !conn->marked_for_close) {
+      resource = TO_DIR_CONN(conn)->requested_resource;
+      if (!strcmpstart(resource, pfx))
+        dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
+                                                  tmp);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
+    fp_pair_map_set(result, fp, (void*)1);
+    tor_free(fp);
+  } SMARTLIST_FOREACH_END(fp);
+
+  smartlist_free(tmp);
+}
+
+/** Launch downloads for all the descriptors whose digests or digests256
+ * are listed as digests[i] for lo <= i < hi.  (Lo and hi may be out of
+ * range.)  If <b>source</b> is given, download from <b>source</b>;
+ * otherwise, download from an appropriate random directory server.
+ */
+MOCK_IMPL(STATIC void,
+initiate_descriptor_downloads,(const routerstatus_t *source,
+                               int purpose, smartlist_t *digests,
+                               int lo, int hi, int pds_flags))
+{
+  char *resource, *cp;
+  int digest_len, enc_digest_len;
+  const char *sep;
+  int b64_256;
+  smartlist_t *tmp;
+
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    /* Microdescriptors are downloaded by "-"-separated base64-encoded
+     * 256-bit digests. */
+    digest_len = DIGEST256_LEN;
+    enc_digest_len = BASE64_DIGEST256_LEN + 1;
+    sep = "-";
+    b64_256 = 1;
+  } else {
+    digest_len = DIGEST_LEN;
+    enc_digest_len = HEX_DIGEST_LEN + 1;
+    sep = "+";
+    b64_256 = 0;
+  }
+
+  if (lo < 0)
+    lo = 0;
+  if (hi > smartlist_len(digests))
+    hi = smartlist_len(digests);
+
+  if (hi-lo <= 0)
+    return;
+
+  tmp = smartlist_new();
+
+  for (; lo < hi; ++lo) {
+    cp = tor_malloc(enc_digest_len);
+    if (b64_256) {
+      digest256_to_base64(cp, smartlist_get(digests, lo));
+    } else {
+      base16_encode(cp, enc_digest_len, smartlist_get(digests, lo),
+                    digest_len);
+    }
+    smartlist_add(tmp, cp);
+  }
+
+  cp = smartlist_join_strings(tmp, sep, 0, NULL);
+  tor_asprintf(&resource, "d/%s.z", cp);
+
+  SMARTLIST_FOREACH(tmp, char *, cp1, tor_free(cp1));
+  smartlist_free(tmp);
+  tor_free(cp);
+
+  if (source) {
+    /* We know which authority or directory mirror we want. */
+    directory_request_t *req = directory_request_new(purpose);
+    directory_request_set_routerstatus(req, source);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
+  } else {
+    directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource,
+                                 pds_flags, DL_WANT_ANY_DIRSERVER);
+  }
+  tor_free(resource);
+}
+
+/** Return the max number of hashes to put in a URL for a given request.
+ */
+static int
+max_dl_per_request(const or_options_t *options, int purpose)
+{
+  /* Since squid does not like URLs >= 4096 bytes we limit it to 96.
+   *   4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535
+   *                 /tor/server/d/.z) == 4026
+   *   4026/41 (40 for the hash and 1 for the + that separates them) => 98
+   *   So use 96 because it's a nice number.
+   *
+   * For microdescriptors, the calculation is
+   *   4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535
+   *                 /tor/micro/d/.z) == 4027
+   *   4027/44 (43 for the hash and 1 for the - that separates them) => 91
+   *   So use 90 because it's a nice number.
+   */
+  int max = 96;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    max = 90;
+  }
+  /* If we're going to tunnel our connections, we can ask for a lot more
+   * in a request. */
+  if (directory_must_use_begindir(options)) {
+    max = 500;
+  }
+  return max;
+}
+
+/** Don't split our requests so finely that we are requesting fewer than
+ * this number per server. (Grouping more than this at once leads to
+ * diminishing returns.) */
+#define MIN_DL_PER_REQUEST 32
+/** To prevent a single screwy cache from confusing us by selective reply,
+ * try to split our requests into at least this many requests. */
+#define MIN_REQUESTS 3
+/** If we want fewer than this many descriptors, wait until we
+ * want more, or until TestingClientMaxIntervalWithoutRequest has passed. */
+#define MAX_DL_TO_DELAY 16
+
+/** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of
+ * router descriptor digests or microdescriptor digest256s in
+ * <b>downloadable</b>, decide whether to delay fetching until we have more.
+ * If we don't want to delay, launch one or more requests to the appropriate
+ * directory authorities.
+ */
+void
+launch_descriptor_downloads(int purpose,
+                            smartlist_t *downloadable,
+                            const routerstatus_t *source, time_t now)
+{
+  const or_options_t *options = get_options();
+  const char *descname;
+  const int fetch_microdesc = (purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  int n_downloadable = smartlist_len(downloadable);
+
+  int i, n_per_request, max_dl_per_req;
+  const char *req_plural = "", *rtr_plural = "";
+  int pds_flags = PDS_RETRY_IF_NO_SERVERS;
+
+  tor_assert(fetch_microdesc || purpose == DIR_PURPOSE_FETCH_SERVERDESC);
+  descname = fetch_microdesc ? "microdesc" : "routerdesc";
+
+  if (!n_downloadable)
+    return;
+
+  if (!directory_fetches_dir_info_early(options)) {
+    if (n_downloadable >= MAX_DL_TO_DELAY) {
+      log_debug(LD_DIR,
+                "There are enough downloadable %ss to launch requests.",
+                descname);
+    } else if (! router_have_minimum_dir_info()) {
+      log_debug(LD_DIR,
+                "We are only missing %d %ss, but we'll fetch anyway, since "
+                "we don't yet have enough directory info.",
+                n_downloadable, descname);
+    } else {
+
+      /* should delay */
+      if ((last_descriptor_download_attempted +
+          options->TestingClientMaxIntervalWithoutRequest) > now)
+        return;
+
+      if (last_descriptor_download_attempted) {
+        log_info(LD_DIR,
+                 "There are not many downloadable %ss, but we've "
+                 "been waiting long enough (%d seconds). Downloading.",
+                 descname,
+                 (int)(now-last_descriptor_download_attempted));
+      } else {
+        log_info(LD_DIR,
+                 "There are not many downloadable %ss, but we haven't "
+                 "tried downloading descriptors recently. Downloading.",
+                 descname);
+      }
+    }
+  }
+
+  if (!authdir_mode(options)) {
+    /* If we wind up going to the authorities, we want to only open one
+     * connection to each authority at a time, so that we don't overload
+     * them.  We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH
+     * regardless of whether we're a cache or not.
+     *
+     * Setting this flag can make initiate_descriptor_downloads() ignore
+     * requests.  We need to make sure that we do in fact call
+     * update_router_descriptor_downloads() later on, once the connections
+     * have succeeded or failed.
+     */
+    pds_flags |= fetch_microdesc ?
+      PDS_NO_EXISTING_MICRODESC_FETCH :
+      PDS_NO_EXISTING_SERVERDESC_FETCH;
+  }
+
+  n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS);
+  max_dl_per_req = max_dl_per_request(options, purpose);
+
+  if (n_per_request > max_dl_per_req)
+    n_per_request = max_dl_per_req;
+
+  if (n_per_request < MIN_DL_PER_REQUEST) {
+    n_per_request = MIN(MIN_DL_PER_REQUEST, n_downloadable);
+  }
+
+  if (n_downloadable > n_per_request)
+    req_plural = rtr_plural = "s";
+  else if (n_downloadable > 1)
+    rtr_plural = "s";
+
+  log_info(LD_DIR,
+           "Launching %d request%s for %d %s%s, %d at a time",
+           CEIL_DIV(n_downloadable, n_per_request), req_plural,
+           n_downloadable, descname, rtr_plural, n_per_request);
+  smartlist_sort_digests(downloadable);
+  for (i=0; i < n_downloadable; i += n_per_request) {
+    initiate_descriptor_downloads(source, purpose,
+                                  downloadable, i, i+n_per_request,
+                                  pds_flags);
+  }
+  last_descriptor_download_attempted = now;
+}
+
+/** For any descriptor that we want that's currently listed in
+ * <b>consensus</b>, download it as appropriate. */
+void
+update_consensus_router_descriptor_downloads(time_t now, int is_vote,
+                                             networkstatus_t *consensus)
+{
+  const or_options_t *options = get_options();
+  digestmap_t *map = NULL;
+  smartlist_t *no_longer_old = smartlist_new();
+  smartlist_t *downloadable = smartlist_new();
+  routerstatus_t *source = NULL;
+  int authdir = authdir_mode(options);
+  int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0,
+    n_inprogress=0, n_in_oldrouters=0;
+
+  if (directory_too_idle_to_fetch_descriptors(options, now))
+    goto done;
+  if (!consensus)
+    goto done;
+
+  if (is_vote) {
+    /* where's it from, so we know whom to ask for descriptors */
+    dir_server_t *ds;
+    networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0);
+    tor_assert(voter);
+    ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest);
+    if (ds)
+      source = &(ds->fake_status);
+    else
+      log_warn(LD_DIR, "couldn't lookup source from vote?");
+  }
+
+  map = digestmap_new();
+  list_pending_descriptor_downloads(map, 0);
+  SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, void *, rsp) {
+      routerstatus_t *rs =
+        is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp;
+      signed_descriptor_t *sd;
+      if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) {
+        const routerinfo_t *ri;
+        ++n_have;
+        if (!(ri = router_get_by_id_digest(rs->identity_digest)) ||
+            tor_memneq(ri->cache_info.signed_descriptor_digest,
+                   sd->signed_descriptor_digest, DIGEST_LEN)) {
+          /* We have a descriptor with this digest, but either there is no
+           * entry in routerlist with the same ID (!ri), or there is one,
+           * but the identity digest differs (memneq).
+           */
+          smartlist_add(no_longer_old, sd);
+          ++n_in_oldrouters; /* We have it in old_routers. */
+        }
+        continue; /* We have it already. */
+      }
+      if (digestmap_get(map, rs->descriptor_digest)) {
+        ++n_inprogress;
+        continue; /* We have an in-progress download. */
+      }
+      if (!download_status_is_ready(&rs->dl_status, now)) {
+        ++n_delayed; /* Not ready for retry. */
+        continue;
+      }
+      if (authdir && dirserv_would_reject_router(rs)) {
+        ++n_would_reject;
+        continue; /* We would throw it out immediately. */
+      }
+      if (!we_want_to_fetch_flavor(options, consensus->flavor) &&
+          !client_would_use_router(rs, now)) {
+        ++n_wouldnt_use;
+        continue; /* We would never use it ourself. */
+      }
+      if (is_vote && source) {
+        char time_bufnew[ISO_TIME_LEN+1];
+        char time_bufold[ISO_TIME_LEN+1];
+        const routerinfo_t *oldrouter;
+        oldrouter = router_get_by_id_digest(rs->identity_digest);
+        format_iso_time(time_bufnew, rs->published_on);
+        if (oldrouter)
+          format_iso_time(time_bufold, oldrouter->cache_info.published_on);
+        log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)",
+                 routerstatus_describe(rs),
+                 time_bufnew,
+                 oldrouter ? time_bufold : "none",
+                 source->nickname, oldrouter ? "known" : "unknown");
+      }
+      smartlist_add(downloadable, rs->descriptor_digest);
+  } SMARTLIST_FOREACH_END(rsp);
+
+  if (!authdir_mode_v3(options)
+      && smartlist_len(no_longer_old)) {
+    routerlist_t *rl = router_get_routerlist();
+    log_info(LD_DIR, "%d router descriptors listed in consensus are "
+             "currently in old_routers; making them current.",
+             smartlist_len(no_longer_old));
+    SMARTLIST_FOREACH_BEGIN(no_longer_old, signed_descriptor_t *, sd) {
+        const char *msg;
+        was_router_added_t r;
+        time_t tmp_cert_expiration_time;
+        routerinfo_t *ri = routerlist_reparse_old(rl, sd);
+        if (!ri) {
+          log_warn(LD_BUG, "Failed to re-parse a router.");
+          continue;
+        }
+        /* need to remember for below, since add_to_routerlist may free. */
+        tmp_cert_expiration_time = ri->cert_expiration_time;
+
+        r = router_add_to_routerlist(ri, &msg, 1, 0);
+        if (WRA_WAS_OUTDATED(r)) {
+          log_warn(LD_DIR, "Couldn't add re-parsed router: %s. This isn't "
+                   "usually a big deal, but you should make sure that your "
+                   "clock and timezone are set correctly.",
+                   msg?msg:"???");
+          if (r == ROUTER_CERTS_EXPIRED) {
+            char time_cons[ISO_TIME_LEN+1];
+            char time_cert_expires[ISO_TIME_LEN+1];
+            format_iso_time(time_cons, consensus->valid_after);
+            format_iso_time(time_cert_expires, tmp_cert_expiration_time);
+            log_warn(LD_DIR, "  (I'm looking at a consensus from %s; This "
+                     "router's certificates began expiring at %s.)",
+                     time_cons, time_cert_expires);
+          }
+        }
+    } SMARTLIST_FOREACH_END(sd);
+    routerlist_assert_ok(rl);
+  }
+
+  log_info(LD_DIR,
+           "%d router descriptors downloadable. %d delayed; %d present "
+           "(%d of those were in old_routers); %d would_reject; "
+           "%d wouldnt_use; %d in progress.",
+           smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters,
+           n_would_reject, n_wouldnt_use, n_inprogress);
+
+  launch_descriptor_downloads(DIR_PURPOSE_FETCH_SERVERDESC,
+                              downloadable, source, now);
+
+  digestmap_free(map, NULL);
+ done:
+  smartlist_free(downloadable);
+  smartlist_free(no_longer_old);
+}
+
+/** How often should we launch a server/authority request to be sure of getting
+ * a guess for our IP? */
+/*XXXX+ this info should come from netinfo cells or something, or we should
+ * do this only when we aren't seeing incoming data. see bug 652. */
+#define DUMMY_DOWNLOAD_INTERVAL (20*60)
+
+/** As needed, launch a dummy router descriptor fetch to see if our
+ * address has changed. */
+static void
+launch_dummy_descriptor_download_as_needed(time_t now,
+                                           const or_options_t *options)
+{
+  static time_t last_dummy_download = 0;
+  /* XXXX+ we could be smarter here; see notes on bug 652. */
+  /* If we're a server that doesn't have a configured address, we rely on
+   * directory fetches to learn when our address changes.  So if we haven't
+   * tried to get any routerdescs in a long time, try a dummy fetch now. */
+  if (!options->Address &&
+      server_mode(options) &&
+      last_descriptor_download_attempted + DUMMY_DOWNLOAD_INTERVAL < now &&
+      last_dummy_download + DUMMY_DOWNLOAD_INTERVAL < now) {
+    last_dummy_download = now;
+    /* XX/teor - do we want an authority here, because they are less likely
+     * to give us the wrong address? (See #17782)
+     * I'm leaving the previous behaviour intact, because I don't like
+     * the idea of some relays contacting an authority every 20 minutes. */
+    directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
+                                 ROUTER_PURPOSE_GENERAL, "authority.z",
+                                 PDS_RETRY_IF_NO_SERVERS,
+                                 DL_WANT_ANY_DIRSERVER);
+  }
+}
+
+/** Launch downloads for router status as needed. */
+void
+update_router_descriptor_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  if (!we_fetch_router_descriptors(options))
+    return;
+
+  update_consensus_router_descriptor_downloads(now, 0,
+                  networkstatus_get_reasonably_live_consensus(now, FLAV_NS));
+}
+
+/** Launch extrainfo downloads as needed. */
+void
+update_extrainfo_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  routerlist_t *rl;
+  smartlist_t *wanted;
+  digestmap_t *pending;
+  int old_routers, i, max_dl_per_req;
+  int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0, n_bogus[2] = {0,0};
+  if (! options->DownloadExtraInfo)
+    return;
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  if (!router_have_minimum_dir_info())
+    return;
+
+  pending = digestmap_new();
+  list_pending_descriptor_downloads(pending, 1);
+  rl = router_get_routerlist();
+  wanted = smartlist_new();
+  for (old_routers = 0; old_routers < 2; ++old_routers) {
+    smartlist_t *lst = old_routers ? rl->old_routers : rl->routers;
+    for (i = 0; i < smartlist_len(lst); ++i) {
+      signed_descriptor_t *sd;
+      char *d;
+      if (old_routers)
+        sd = smartlist_get(lst, i);
+      else
+        sd = &((routerinfo_t*)smartlist_get(lst, i))->cache_info;
+      if (sd->is_extrainfo)
+        continue; /* This should never happen. */
+      if (old_routers && !router_get_by_id_digest(sd->identity_digest))
+        continue; /* Couldn't check the signature if we got it. */
+      if (sd->extrainfo_is_bogus)
+        continue;
+      d = sd->extra_info_digest;
+      if (tor_digest_is_zero(d)) {
+        ++n_no_ei;
+        continue;
+      }
+      if (eimap_get(rl->extra_info_map, d)) {
+        ++n_have;
+        continue;
+      }
+      if (!download_status_is_ready(&sd->ei_dl_status, now)) {
+        ++n_delay;
+        continue;
+      }
+      if (digestmap_get(pending, d)) {
+        ++n_pending;
+        continue;
+      }
+
+      const signed_descriptor_t *sd2 = router_get_by_extrainfo_digest(d);
+      if (sd2 != sd) {
+        if (sd2 != NULL) {
+          char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1];
+          char d3[HEX_DIGEST_LEN+1], d4[HEX_DIGEST_LEN+1];
+          base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN);
+          base16_encode(d2, sizeof(d2), sd2->identity_digest, DIGEST_LEN);
+          base16_encode(d3, sizeof(d3), d, DIGEST_LEN);
+          base16_encode(d4, sizeof(d3), sd2->extra_info_digest, DIGEST_LEN);
+
+          log_info(LD_DIR, "Found an entry in %s with mismatched "
+                   "router_get_by_extrainfo_digest() value. This has ID %s "
+                   "but the entry in the map has ID %s. This has EI digest "
+                   "%s and the entry in the map has EI digest %s.",
+                   old_routers?"old_routers":"routers",
+                   d1, d2, d3, d4);
+        } else {
+          char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1];
+          base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN);
+          base16_encode(d2, sizeof(d2), d, DIGEST_LEN);
+
+          log_info(LD_DIR, "Found an entry in %s with NULL "
+                   "router_get_by_extrainfo_digest() value. This has ID %s "
+                   "and EI digest %s.",
+                   old_routers?"old_routers":"routers",
+                   d1, d2);
+        }
+        ++n_bogus[old_routers];
+        continue;
+      }
+      smartlist_add(wanted, d);
+    }
+  }
+  digestmap_free(pending, NULL);
+
+  log_info(LD_DIR, "Extrainfo download status: %d router with no ei, %d "
+           "with present ei, %d delaying, %d pending, %d downloadable, %d "
+           "bogus in routers, %d bogus in old_routers",
+           n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted),
+           n_bogus[0], n_bogus[1]);
+
+  smartlist_shuffle(wanted);
+
+  max_dl_per_req = max_dl_per_request(options, DIR_PURPOSE_FETCH_EXTRAINFO);
+  for (i = 0; i < smartlist_len(wanted); i += max_dl_per_req) {
+    initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO,
+                                  wanted, i, i+max_dl_per_req,
+                PDS_RETRY_IF_NO_SERVERS|PDS_NO_EXISTING_SERVERDESC_FETCH);
+  }
+
+  smartlist_free(wanted);
+}
+
+/** Reset the consensus and extra-info download failure count on all routers.
+ * When we get a new consensus,
+ * routers_update_status_from_consensus_networkstatus() will reset the
+ * download statuses on the descriptors in that consensus.
+ */
+void
+router_reset_descriptor_download_failures(void)
+{
+  log_debug(LD_GENERAL,
+            "In router_reset_descriptor_download_failures()");
+
+  networkstatus_reset_download_failures();
+  last_descriptor_download_attempted = 0;
+  if (!routerlist)
+    return;
+  /* We want to download *all* extra-info descriptors, not just those in
+   * the consensus we currently have (or are about to have) */
+  SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
+  {
+    download_status_reset(&ri->cache_info.ei_dl_status);
+  });
+  SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd,
+  {
+    download_status_reset(&sd->ei_dl_status);
+  });
+}
+
+/** Any changes in a router descriptor's publication time larger than this are
+ * automatically non-cosmetic. */
+#define ROUTER_MAX_COSMETIC_TIME_DIFFERENCE (2*60*60)
+
+/** We allow uptime to vary from how much it ought to be by this much. */
+#define ROUTER_ALLOW_UPTIME_DRIFT (6*60*60)
+
+/** Return true iff the only differences between r1 and r2 are such that
+ * would not cause a recent (post 0.1.1.6) dirserver to republish.
+ */
+int
+router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2)
+{
+  time_t r1pub, r2pub;
+  long time_difference;
+  tor_assert(r1 && r2);
+
+  /* r1 should be the one that was published first. */
+  if (r1->cache_info.published_on > r2->cache_info.published_on) {
+    const routerinfo_t *ri_tmp = r2;
+    r2 = r1;
+    r1 = ri_tmp;
+  }
+
+  /* If any key fields differ, they're different. */
+  if (r1->addr != r2->addr ||
+      strcasecmp(r1->nickname, r2->nickname) ||
+      r1->or_port != r2->or_port ||
+      !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) ||
+      r1->ipv6_orport != r2->ipv6_orport ||
+      r1->dir_port != r2->dir_port ||
+      r1->purpose != r2->purpose ||
+      !crypto_pk_eq_keys(r1->onion_pkey, r2->onion_pkey) ||
+      !crypto_pk_eq_keys(r1->identity_pkey, r2->identity_pkey) ||
+      strcasecmp(r1->platform, r2->platform) ||
+      (r1->contact_info && !r2->contact_info) || /* contact_info is optional */
+      (!r1->contact_info && r2->contact_info) ||
+      (r1->contact_info && r2->contact_info &&
+       strcasecmp(r1->contact_info, r2->contact_info)) ||
+      r1->is_hibernating != r2->is_hibernating ||
+      ! addr_policies_eq(r1->exit_policy, r2->exit_policy) ||
+      (r1->supports_tunnelled_dir_requests !=
+       r2->supports_tunnelled_dir_requests))
+    return 0;
+  if ((r1->declared_family == NULL) != (r2->declared_family == NULL))
+    return 0;
+  if (r1->declared_family && r2->declared_family) {
+    int i, n;
+    if (smartlist_len(r1->declared_family)!=smartlist_len(r2->declared_family))
+      return 0;
+    n = smartlist_len(r1->declared_family);
+    for (i=0; i < n; ++i) {
+      if (strcasecmp(smartlist_get(r1->declared_family, i),
+                     smartlist_get(r2->declared_family, i)))
+        return 0;
+    }
+  }
+
+  /* Did bandwidth change a lot? */
+  if ((r1->bandwidthcapacity < r2->bandwidthcapacity/2) ||
+      (r2->bandwidthcapacity < r1->bandwidthcapacity/2))
+    return 0;
+
+  /* Did the bandwidthrate or bandwidthburst change? */
+  if ((r1->bandwidthrate != r2->bandwidthrate) ||
+      (r1->bandwidthburst != r2->bandwidthburst))
+    return 0;
+
+  /* Did more than 12 hours pass? */
+  if (r1->cache_info.published_on + ROUTER_MAX_COSMETIC_TIME_DIFFERENCE
+      < r2->cache_info.published_on)
+    return 0;
+
+  /* Did uptime fail to increase by approximately the amount we would think,
+   * give or take some slop? */
+  r1pub = r1->cache_info.published_on;
+  r2pub = r2->cache_info.published_on;
+  time_difference = labs(r2->uptime - (r1->uptime + (r2pub - r1pub)));
+  if (time_difference > ROUTER_ALLOW_UPTIME_DRIFT &&
+      time_difference > r1->uptime * .05 &&
+      time_difference > r2->uptime * .05)
+    return 0;
+
+  /* Otherwise, the difference is cosmetic. */
+  return 1;
+}
+
+/** Check whether <b>sd</b> describes a router descriptor compatible with the
+ * extrainfo document <b>ei</b>.
+ *
+ * <b>identity_pkey</b> (which must also be provided) is RSA1024 identity key
+ * for the router. We use it to check the signature of the extrainfo document,
+ * if it has not already been checked.
+ *
+ * If no router is compatible with <b>ei</b>, <b>ei</b> should be
+ * dropped.  Return 0 for "compatible", return 1 for "reject, and inform
+ * whoever uploaded <b>ei</b>, and return -1 for "reject silently.".  If
+ * <b>msg</b> is present, set *<b>msg</b> to a description of the
+ * incompatibility (if any).
+ *
+ * Set the extrainfo_is_bogus field in <b>sd</b> if the digests matched
+ * but the extrainfo was nonetheless incompatible.
+ **/
+int
+routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey,
+                                       extrainfo_t *ei,
+                                       signed_descriptor_t *sd,
+                                       const char **msg)
+{
+  int digest_matches, digest256_matches, r=1;
+  tor_assert(identity_pkey);
+  tor_assert(sd);
+  tor_assert(ei);
+
+  if (ei->bad_sig) {
+    if (msg) *msg = "Extrainfo signature was bad, or signed with wrong key.";
+    return 1;
+  }
+
+  digest_matches = tor_memeq(ei->cache_info.signed_descriptor_digest,
+                           sd->extra_info_digest, DIGEST_LEN);
+  /* Set digest256_matches to 1 if the digest is correct, or if no
+   * digest256 was in the ri. */
+  digest256_matches = tor_memeq(ei->digest256,
+                                sd->extra_info_digest256, DIGEST256_LEN);
+  digest256_matches |=
+    tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
+
+  /* The identity must match exactly to have been generated at the same time
+   * by the same router. */
+  if (tor_memneq(sd->identity_digest,
+                 ei->cache_info.identity_digest,
+                 DIGEST_LEN)) {
+    if (msg) *msg = "Extrainfo nickname or identity did not match routerinfo";
+    goto err; /* different servers */
+  }
+
+  if (! tor_cert_opt_eq(sd->signing_key_cert,
+                        ei->cache_info.signing_key_cert)) {
+    if (msg) *msg = "Extrainfo signing key cert didn't match routerinfo";
+    goto err; /* different servers */
+  }
+
+  if (ei->pending_sig) {
+    char signed_digest[128];
+    if (crypto_pk_public_checksig(identity_pkey,
+                       signed_digest, sizeof(signed_digest),
+                       ei->pending_sig, ei->pending_sig_len) != DIGEST_LEN ||
+        tor_memneq(signed_digest, ei->cache_info.signed_descriptor_digest,
+               DIGEST_LEN)) {
+      ei->bad_sig = 1;
+      tor_free(ei->pending_sig);
+      if (msg) *msg = "Extrainfo signature bad, or signed with wrong key";
+      goto err; /* Bad signature, or no match. */
+    }
+
+    ei->cache_info.send_unencrypted = sd->send_unencrypted;
+    tor_free(ei->pending_sig);
+  }
+
+  if (ei->cache_info.published_on < sd->published_on) {
+    if (msg) *msg = "Extrainfo published time did not match routerdesc";
+    goto err;
+  } else if (ei->cache_info.published_on > sd->published_on) {
+    if (msg) *msg = "Extrainfo published time did not match routerdesc";
+    r = -1;
+    goto err;
+  }
+
+  if (!digest256_matches && !digest_matches) {
+    if (msg) *msg = "Neither digest256 or digest matched "
+               "digest from routerdesc";
+    goto err;
+  }
+
+  if (!digest256_matches) {
+    if (msg) *msg = "Extrainfo digest did not match digest256 from routerdesc";
+    goto err; /* Digest doesn't match declared value. */
+  }
+
+  if (!digest_matches) {
+    if (msg) *msg = "Extrainfo digest did not match value from routerdesc";
+    goto err; /* Digest doesn't match declared value. */
+  }
+
+  return 0;
+ err:
+  if (digest_matches) {
+    /* This signature was okay, and the digest was right: This is indeed the
+     * corresponding extrainfo.  But insanely, it doesn't match the routerinfo
+     * that lists it.  Don't try to fetch this one again. */
+    sd->extrainfo_is_bogus = 1;
+  }
+
+  return r;
+}
+
+/* Does ri have a valid ntor onion key?
+ * Valid ntor onion keys exist and have at least one non-zero byte. */
+int
+routerinfo_has_curve25519_onion_key(const routerinfo_t *ri)
+{
+  if (!ri) {
+    return 0;
+  }
+
+  if (!ri->onion_curve25519_pkey) {
+    return 0;
+  }
+
+  if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
+                      CURVE25519_PUBKEY_LEN)) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* Is rs running a tor version known to support EXTEND2 cells?
+ * If allow_unknown_versions is true, return true if we can't tell
+ * (from a versions line or a protocols line) whether it supports extend2
+ * cells.
+ * Otherwise, return false if the version is unknown. */
+int
+routerstatus_version_supports_extend2_cells(const routerstatus_t *rs,
+                                            int allow_unknown_versions)
+{
+  if (!rs) {
+    return allow_unknown_versions;
+  }
+
+  if (!rs->pv.protocols_known) {
+    return allow_unknown_versions;
+  }
+
+  return rs->pv.supports_extend2_cells;
+}
+
+/** Assert that the internal representation of <b>rl</b> is
+ * self-consistent. */
+void
+routerlist_assert_ok(const routerlist_t *rl)
+{
+  routerinfo_t *r2;
+  signed_descriptor_t *sd2;
+  if (!rl)
+    return;
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, r) {
+    r2 = rimap_get(rl->identity_map, r->cache_info.identity_digest);
+    tor_assert(r == r2);
+    sd2 = sdmap_get(rl->desc_digest_map,
+                    r->cache_info.signed_descriptor_digest);
+    tor_assert(&(r->cache_info) == sd2);
+    tor_assert(r->cache_info.routerlist_index == r_sl_idx);
+    /* XXXX
+     *
+     *   Hoo boy.  We need to fix this one, and the fix is a bit tricky, so
+     * commenting this out is just a band-aid.
+     *
+     *   The problem is that, although well-behaved router descriptors
+     * should never have the same value for their extra_info_digest, it's
+     * possible for ill-behaved routers to claim whatever they like there.
+     *
+     *   The real answer is to trash desc_by_eid_map and instead have
+     * something that indicates for a given extra-info digest we want,
+     * what its download status is.  We'll do that as a part of routerlist
+     * refactoring once consensus directories are in.  For now,
+     * this rep violation is probably harmless: an adversary can make us
+     * reset our retry count for an extrainfo, but that's not the end
+     * of the world.  Changing the representation in 0.2.0.x would just
+     * destabilize the codebase.
+    if (!tor_digest_is_zero(r->cache_info.extra_info_digest)) {
+      signed_descriptor_t *sd3 =
+        sdmap_get(rl->desc_by_eid_map, r->cache_info.extra_info_digest);
+      tor_assert(sd3 == &(r->cache_info));
+    }
+    */
+  } SMARTLIST_FOREACH_END(r);
+  SMARTLIST_FOREACH_BEGIN(rl->old_routers, signed_descriptor_t *, sd) {
+    r2 = rimap_get(rl->identity_map, sd->identity_digest);
+    tor_assert(!r2 || sd != &(r2->cache_info));
+    sd2 = sdmap_get(rl->desc_digest_map, sd->signed_descriptor_digest);
+    tor_assert(sd == sd2);
+    tor_assert(sd->routerlist_index == sd_sl_idx);
+    /* XXXX see above.
+    if (!tor_digest_is_zero(sd->extra_info_digest)) {
+      signed_descriptor_t *sd3 =
+        sdmap_get(rl->desc_by_eid_map, sd->extra_info_digest);
+      tor_assert(sd3 == sd);
+    }
+    */
+  } SMARTLIST_FOREACH_END(sd);
+
+  RIMAP_FOREACH(rl->identity_map, d, r) {
+    tor_assert(tor_memeq(r->cache_info.identity_digest, d, DIGEST_LEN));
+  } DIGESTMAP_FOREACH_END;
+  SDMAP_FOREACH(rl->desc_digest_map, d, sd) {
+    tor_assert(tor_memeq(sd->signed_descriptor_digest, d, DIGEST_LEN));
+  } DIGESTMAP_FOREACH_END;
+  SDMAP_FOREACH(rl->desc_by_eid_map, d, sd) {
+    tor_assert(!tor_digest_is_zero(d));
+    tor_assert(sd);
+    tor_assert(tor_memeq(sd->extra_info_digest, d, DIGEST_LEN));
+  } DIGESTMAP_FOREACH_END;
+  EIMAP_FOREACH(rl->extra_info_map, d, ei) {
+    signed_descriptor_t *sd;
+    tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest,
+                       d, DIGEST_LEN));
+    sd = sdmap_get(rl->desc_by_eid_map,
+                   ei->cache_info.signed_descriptor_digest);
+    // tor_assert(sd); // XXXX see above
+    if (sd) {
+      tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest,
+                         sd->extra_info_digest, DIGEST_LEN));
+    }
+  } DIGESTMAP_FOREACH_END;
+}
+
+/** Allocate and return a new string representing the contact info
+ * and platform string for <b>router</b>,
+ * surrounded by quotes and using standard C escapes.
+ *
+ * THIS FUNCTION IS NOT REENTRANT.  Don't call it from outside the main
+ * thread.  Also, each call invalidates the last-returned value, so don't
+ * try log_warn(LD_GENERAL, "%s %s", esc_router_info(a), esc_router_info(b));
+ *
+ * If <b>router</b> is NULL, it just frees its internal memory and returns.
+ */
+const char *
+esc_router_info(const routerinfo_t *router)
+{
+  static char *info=NULL;
+  char *esc_contact, *esc_platform;
+  tor_free(info);
+
+  if (!router)
+    return NULL; /* we're exiting; just free the memory we use */
+
+  esc_contact = esc_for_log(router->contact_info);
+  esc_platform = esc_for_log(router->platform);
+
+  tor_asprintf(&info, "Contact %s, Platform %s", esc_contact, esc_platform);
+  tor_free(esc_contact);
+  tor_free(esc_platform);
+
+  return info;
+}
+
+/** Helper for sorting: compare two routerinfos by their identity
+ * digest. */
+static int
+compare_routerinfo_by_id_digest_(const void **a, const void **b)
+{
+  routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b;
+  return fast_memcmp(first->cache_info.identity_digest,
+                second->cache_info.identity_digest,
+                DIGEST_LEN);
+}
+
+/** Sort a list of routerinfo_t in ascending order of identity digest. */
+void
+routers_sort_by_identity(smartlist_t *routers)
+{
+  smartlist_sort(routers, compare_routerinfo_by_id_digest_);
+}
+
+/** Called when we change a node set, or when we reload the geoip IPv4 list:
+ * recompute all country info in all configuration node sets and in the
+ * routerlist. */
+void
+refresh_all_country_info(void)
+{
+  const or_options_t *options = get_options();
+
+  if (options->EntryNodes)
+    routerset_refresh_countries(options->EntryNodes);
+  if (options->ExitNodes)
+    routerset_refresh_countries(options->ExitNodes);
+  if (options->ExcludeNodes)
+    routerset_refresh_countries(options->ExcludeNodes);
+  if (options->ExcludeExitNodes)
+    routerset_refresh_countries(options->ExcludeExitNodes);
+  if (options->ExcludeExitNodesUnion_)
+    routerset_refresh_countries(options->ExcludeExitNodesUnion_);
+
+  nodelist_refresh_countries();
+}
diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h
new file mode 100644
index 0000000000..4b7406364f
--- /dev/null
+++ b/src/feature/nodelist/routerlist.h
@@ -0,0 +1,347 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerlist.h
+ * \brief Header file for routerlist.c.
+ **/
+
+#ifndef TOR_ROUTERLIST_H
+#define TOR_ROUTERLIST_H
+
+#include "lib/testsupport/testsupport.h"
+
+/** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */
+typedef enum was_router_added_t {
+  /* Router was added successfully. */
+  ROUTER_ADDED_SUCCESSFULLY = 1,
+  /* Extrainfo document was rejected because no corresponding router
+   * descriptor was found OR router descriptor was rejected because
+   * it was incompatible with its extrainfo document. */
+  ROUTER_BAD_EI = -1,
+  /* Router descriptor was rejected because it is already known. */
+  ROUTER_IS_ALREADY_KNOWN = -2,
+  /* General purpose router was rejected, because it was not listed
+   * in consensus. */
+  ROUTER_NOT_IN_CONSENSUS = -3,
+  /* Router was neither in directory consensus nor in any of
+   * networkstatus documents. Caching it to access later.
+   * (Applies to fetched descriptors only.) */
+  ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4,
+  /* Router was rejected by directory authority. */
+  ROUTER_AUTHDIR_REJECTS = -5,
+  /* Bridge descriptor was rejected because such bridge was not one
+   * of the bridges we have listed in our configuration. */
+  ROUTER_WAS_NOT_WANTED = -6,
+  /* Router descriptor was rejected because it was older than
+   * OLD_ROUTER_DESC_MAX_AGE. */
+  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
+  /* DOCDOC */
+  ROUTER_CERTS_EXPIRED = -8
+} was_router_added_t;
+
+/** Flags to be passed to control router_choose_random_node() to indicate what
+ * kind of nodes to pick according to what algorithm. */
+typedef enum router_crn_flags_t {
+  CRN_NEED_UPTIME = 1<<0,
+  CRN_NEED_CAPACITY = 1<<1,
+  CRN_NEED_GUARD = 1<<2,
+  /* XXXX not used, apparently. */
+  CRN_WEIGHT_AS_EXIT = 1<<5,
+  CRN_NEED_DESC = 1<<6,
+  /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
+  CRN_PREF_ADDR = 1<<7,
+  /* On clients, only provide nodes that we can connect to directly, based on
+   * our firewall rules */
+  CRN_DIRECT_CONN = 1<<8,
+  /* On clients, only provide nodes with HSRend >= 2 protocol version which
+   * is required for hidden service version >= 3. */
+  CRN_RENDEZVOUS_V3 = 1<<9,
+} router_crn_flags_t;
+
+/** Possible ways to weight routers when choosing one randomly.  See
+ * routerlist_sl_choose_by_bandwidth() for more information.*/
+typedef enum bandwidth_weight_rule_t {
+  NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
+  WEIGHT_FOR_DIR
+} bandwidth_weight_rule_t;
+
+/* Flags for pick_directory_server() and pick_trusteddirserver(). */
+/** Flag to indicate that we should not automatically be willing to use
+ * ourself to answer a directory request.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_ALLOW_SELF                 (1<<0)
+/** Flag to indicate that if no servers seem to be up, we should mark all
+ * directory servers as up and try again.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_RETRY_IF_NO_SERVERS        (1<<1)
+/** Flag to indicate that we should not exclude directory servers that
+ * our ReachableAddress settings would exclude.  This usually means that
+ * we're going to connect to the server over Tor, and so we don't need to
+ * worry about our firewall telling us we can't.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_IGNORE_FASCISTFIREWALL     (1<<2)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading server descriptors
+ * or extrainfo documents.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading microdescs.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
+
+int get_n_authorities(dirinfo_type_t type);
+int trusted_dirs_reload_certs(void);
+
+/*
+ * Pass one of these as source to trusted_dirs_load_certs_from_string()
+ * to indicate whence string originates; this controls error handling
+ * behavior such as marking downloads as failed.
+ */
+
+#define TRUSTED_DIRS_CERTS_SRC_SELF 0
+#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
+#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
+
+int trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                        int flush, const char *source_dir);
+void trusted_dirs_flush_certs_to_disk(void);
+authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
+authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
+authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
+                                                const char *sk_digest);
+void authority_cert_get_all(smartlist_t *certs_out);
+void authority_cert_dl_failed(const char *id_digest,
+                              const char *signing_key_digest, int status);
+void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                                   const char *dir_hint);
+int router_reload_router_list(void);
+int authority_cert_dl_looks_uncertain(const char *id_digest);
+const smartlist_t *router_get_trusted_dir_servers(void);
+const smartlist_t *router_get_fallback_dir_servers(void);
+int authority_cert_is_blacklisted(const authority_cert_t *cert);
+
+const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
+                                                   int flags);
+dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
+dir_server_t *router_get_fallback_dirserver_by_digest(
+                                                   const char *digest);
+int router_digest_is_fallback_dir(const char *digest);
+MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
+          (const char *d));
+const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
+                                                   int flags);
+const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
+                                                     int flags);
+int router_skip_or_reachability(const or_options_t *options, int try_ip_pref);
+int router_get_my_share_of_directory_requests(double *v3_share_out);
+void router_reset_status_download_failures(void);
+int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2);
+void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                           int need_capacity, int need_guard,
+                                           int need_desc, int pref_addr,
+                                           int direct_conn);
+
+const routerinfo_t *routerlist_find_my_routerinfo(void);
+uint32_t router_get_advertised_bandwidth(const routerinfo_t *router);
+uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router);
+
+const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                                          bandwidth_weight_rule_t rule);
+double frac_nodes_with_descriptors(const smartlist_t *sl,
+                                   bandwidth_weight_rule_t rule,
+                                   int for_direct_conn);
+
+const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
+                                        struct routerset_t *excludedset,
+                                        router_crn_flags_t flags);
+
+int router_digest_is_trusted_dir_type(const char *digest,
+                                      dirinfo_type_t type);
+#define router_digest_is_trusted_dir(d) \
+  router_digest_is_trusted_dir_type((d), NO_DIRINFO)
+
+int hexdigest_to_digest(const char *hexdigest, char *digest);
+const routerinfo_t *router_get_by_id_digest(const char *digest);
+routerinfo_t *router_get_mutable_by_digest(const char *digest);
+signed_descriptor_t *router_get_by_descriptor_digest(const char *digest);
+MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest,
+          (const char *digest));
+MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest,
+          (const char *digest));
+const char *signed_descriptor_get_body(const signed_descriptor_t *desc);
+const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc);
+routerlist_t *router_get_routerlist(void);
+void routerinfo_free_(routerinfo_t *router);
+#define routerinfo_free(router) \
+  FREE_AND_NULL(routerinfo_t, routerinfo_free_, (router))
+void extrainfo_free_(extrainfo_t *extrainfo);
+#define extrainfo_free(ei) FREE_AND_NULL(extrainfo_t, extrainfo_free_, (ei))
+void routerlist_free_(routerlist_t *rl);
+#define routerlist_free(rl) FREE_AND_NULL(routerlist_t, routerlist_free_, (rl))
+void dump_routerlist_mem_usage(int severity);
+void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old,
+                       time_t now);
+void routerlist_free_all(void);
+void routerlist_reset_warnings(void);
+
+MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
+MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
+          (const char *digest));
+MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
+          (const char *digest));
+MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
+    (const char *id_digest, const char *sk_digest));
+
+static int WRA_WAS_ADDED(was_router_added_t s);
+static int WRA_WAS_OUTDATED(was_router_added_t s);
+static int WRA_WAS_REJECTED(was_router_added_t s);
+static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s);
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was added. It might still be necessary to check whether the descriptor
+ * generator should be notified.
+ */
+static inline int
+WRA_WAS_ADDED(was_router_added_t s) {
+  return s == ROUTER_ADDED_SUCCESSFULLY;
+}
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was not added because it was either:
+ * - not in the consensus
+ * - neither in the consensus nor in any networkstatus document
+ * - it was outdated.
+ * - its certificates were expired.
+ */
+static inline int WRA_WAS_OUTDATED(was_router_added_t s)
+{
+  return (s == ROUTER_WAS_TOO_OLD ||
+          s == ROUTER_IS_ALREADY_KNOWN ||
+          s == ROUTER_NOT_IN_CONSENSUS ||
+          s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS ||
+          s == ROUTER_CERTS_EXPIRED);
+}
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was flat-out rejected. */
+static inline int WRA_WAS_REJECTED(was_router_added_t s)
+{
+  return (s == ROUTER_AUTHDIR_REJECTS);
+}
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was flat-out rejected. */
+static inline int WRA_NEVER_DOWNLOADABLE(was_router_added_t s)
+{
+  return (s == ROUTER_AUTHDIR_REJECTS ||
+          s == ROUTER_BAD_EI ||
+          s == ROUTER_WAS_TOO_OLD ||
+          s == ROUTER_CERTS_EXPIRED);
+}
+was_router_added_t router_add_to_routerlist(routerinfo_t *router,
+                                            const char **msg,
+                                            int from_cache,
+                                            int from_fetch);
+was_router_added_t router_add_extrainfo_to_routerlist(
+                                        extrainfo_t *ei, const char **msg,
+                                        int from_cache, int from_fetch);
+void routerlist_descriptors_added(smartlist_t *sl, int from_cache);
+void routerlist_remove_old_routers(void);
+int router_load_single_router(const char *s, uint8_t purpose, int cache,
+                              const char **msg);
+int router_load_routers_from_string(const char *s, const char *eos,
+                                     saved_location_t saved_location,
+                                     smartlist_t *requested_fingerprints,
+                                     int descriptor_digests,
+                                     const char *prepend_annotations);
+void router_load_extrainfo_from_string(const char *s, const char *eos,
+                                       saved_location_t saved_location,
+                                       smartlist_t *requested_fingerprints,
+                                       int descriptor_digests);
+
+void routerlist_retry_directory_downloads(time_t now);
+
+int router_exit_policy_rejects_all(const routerinfo_t *router);
+
+dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *addrport_ipv6,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight);
+dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
+                                      uint16_t dir_port, uint16_t or_port,
+                                      const tor_addr_port_t *addrport_ipv6,
+                                      const char *id_digest, double weight);
+void dir_server_add(dir_server_t *ent);
+
+void authority_cert_free_(authority_cert_t *cert);
+#define authority_cert_free(cert) \
+  FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
+void clear_dir_servers(void);
+void update_consensus_router_descriptor_downloads(time_t now, int is_vote,
+                                                  networkstatus_t *consensus);
+void update_router_descriptor_downloads(time_t now);
+void update_all_descriptor_downloads(time_t now);
+void update_extrainfo_downloads(time_t now);
+void router_reset_descriptor_download_failures(void);
+int router_differences_are_cosmetic(const routerinfo_t *r1,
+                                    const routerinfo_t *r2);
+int routerinfo_incompatible_with_extrainfo(const crypto_pk_t *ri,
+                                           extrainfo_t *ei,
+                                           signed_descriptor_t *sd,
+                                           const char **msg);
+int routerinfo_has_curve25519_onion_key(const routerinfo_t *ri);
+int routerstatus_version_supports_extend2_cells(const routerstatus_t *rs,
+                                                int allow_unknown_versions);
+
+void routerlist_assert_ok(const routerlist_t *rl);
+const char *esc_router_info(const routerinfo_t *router);
+void routers_sort_by_identity(smartlist_t *routers);
+
+void refresh_all_country_info(void);
+
+void list_pending_microdesc_downloads(digest256map_t *result);
+void launch_descriptor_downloads(int purpose,
+                                 smartlist_t *downloadable,
+                                 const routerstatus_t *source,
+                                 time_t now);
+
+int hex_digest_nickname_decode(const char *hexdigest,
+                               char *digest_out,
+                               char *nickname_qualifier_out,
+                               char *nickname_out);
+int hex_digest_nickname_matches(const char *hexdigest,
+                                const char *identity_digest,
+                                const char *nickname);
+
+#ifdef ROUTERLIST_PRIVATE
+STATIC int choose_array_element_by_weight(const uint64_t *entries,
+                                          int n_entries);
+STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
+                                        const double *entries_in,
+                                        int n_entries,
+                                        uint64_t *total_out);
+STATIC const routerstatus_t *router_pick_directory_server_impl(
+                                           dirinfo_type_t auth, int flags,
+                                           int *n_busy_out);
+
+MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router,
+                                                 int seconds));
+MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
+          (routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible));
+
+MOCK_DECL(STATIC void, initiate_descriptor_downloads,
+          (const routerstatus_t *source, int purpose, smartlist_t *digests,
+           int lo, int hi, int pds_flags));
+STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
+                                          int serverdesc, int microdesc);
+
+#endif /* defined(ROUTERLIST_PRIVATE) */
+
+#endif /* !defined(TOR_ROUTERLIST_H) */
diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h
new file mode 100644
index 0000000000..0b94a4dfcd
--- /dev/null
+++ b/src/feature/nodelist/routerlist_st.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERLIST_ST_H
+#define ROUTERLIST_ST_H
+
+#include "or/desc_store_st.h"
+
+/** Contents of a directory of onion routers. */
+struct routerlist_t {
+  /** Map from server identity digest to a member of routers. */
+  struct digest_ri_map_t *identity_map;
+  /** Map from server descriptor digest to a signed_descriptor_t from
+   * routers or old_routers. */
+  struct digest_sd_map_t *desc_digest_map;
+  /** Map from extra-info digest to an extrainfo_t.  Only exists for
+   * routers in routers or old_routers. */
+  struct digest_ei_map_t *extra_info_map;
+  /** Map from extra-info digests to a signed_descriptor_t for a router
+   * descriptor having that extra-info digest.  Only exists for
+   * routers in routers or old_routers. */
+  struct digest_sd_map_t *desc_by_eid_map;
+  /** List of routerinfo_t for all currently live routers we know. */
+  smartlist_t *routers;
+  /** List of signed_descriptor_t for older router descriptors we're
+   * caching. */
+  smartlist_t *old_routers;
+  /** Store holding server descriptors.  If present, any router whose
+   * cache_info.saved_location == SAVED_IN_CACHE is stored in this file
+   * starting at cache_info.saved_offset */
+  desc_store_t desc_store;
+  /** Store holding extra-info documents. */
+  desc_store_t extrainfo_store;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c
new file mode 100644
index 0000000000..273666046b
--- /dev/null
+++ b/src/feature/nodelist/routerparse.c
@@ -0,0 +1,5693 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerparse.c
+ * \brief Code to parse and validate router descriptors, consenus directories,
+ *   and similar objects.
+ *
+ * The objects parsed by this module use a common text-based metaformat,
+ * documented in dir-spec.txt in torspec.git.  This module is itself divided
+ * into two major kinds of function: code to handle the metaformat, and code
+ * to convert from particular instances of the metaformat into the
+ * objects that Tor uses.
+ *
+ * The generic parsing code works by calling a table-based tokenizer on the
+ * input string.  Each token corresponds to a single line with a token, plus
+ * optional arguments on that line, plus an optional base-64 encoded object
+ * after that line.  Each token has a definition in a table of token_rule_t
+ * entries that describes how many arguments it can take, whether it takes an
+ * object, how many times it may appear, whether it must appear first, and so
+ * on.
+ *
+ * The tokenizer function tokenize_string() converts its string input into a
+ * smartlist full of instances of directory_token_t, according to a provided
+ * table of token_rule_t.
+ *
+ * The generic parts of this module additionally include functions for
+ * finding the start and end of signed information inside a signed object, and
+ * computing the digest that will be signed.
+ *
+ * There are also functions for saving objects to disk that have caused
+ * parsing to fail.
+ *
+ * The specific parts of this module describe conversions between
+ * particular lists of directory_token_t and particular objects.  The
+ * kinds of objects that can be parsed here are:
+ *  <ul>
+ *  <li>router descriptors (managed from routerlist.c)
+ *  <li>extra-info documents (managed from routerlist.c)
+ *  <li>microdescriptors (managed from microdesc.c)
+ *  <li>vote and consensus networkstatus documents, and the routerstatus_t
+ *    objects that they comprise (managed from networkstatus.c)
+ *  <li>detached-signature objects used by authorities for gathering
+ *    signatures on the networkstatus consensus (managed from dirvote.c)
+ *  <li>authority key certificates (managed from routerlist.c)
+ *  <li>hidden service descriptors (managed from rendcommon.c and rendcache.c)
+ * </ul>
+ *
+ * For no terribly good reason, the functions to <i>generate</i> signatures on
+ * the above directory objects are also in this module.
+ **/
+
+#define ROUTERPARSE_PRIVATE
+
+#include "or/or.h"
+#include "or/circuitstats.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/dirauth/shared_random.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "lib/memarea/memarea.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/parsecommon.h"
+#include "or/policies.h"
+#include "or/protover.h"
+#include "or/rendcommon.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerkeys.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "lib/sandbox/sandbox.h"
+#include "or/shared_random_client.h"
+#include "or/torcert.h"
+#include "or/voting_schedule.h"
+
+#include "or/dirauth/dirvote.h"
+
+#include "or/addr_policy_st.h"
+#include "or/authority_cert_st.h"
+#include "or/document_signature_st.h"
+#include "or/extend_info_st.h"
+#include "or/extrainfo_st.h"
+#include "or/microdesc_st.h"
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+#include "or/ns_detached_signatures_st.h"
+#include "or/rend_authorized_client_st.h"
+#include "or/rend_intro_point_st.h"
+#include "or/rend_service_descriptor_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/tor_version_st.h"
+#include "or/vote_microdesc_hash_st.h"
+#include "or/vote_routerstatus_st.h"
+
+#include "lib/container/bloomfilt.h"
+
+#undef log
+#include <math.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+/****************************************************************************/
+
+/** List of tokens recognized in router descriptors */
+static token_rule_t routerdesc_token_table[] = {
+  T0N("reject",              K_REJECT,              ARGS,    NO_OBJ ),
+  T0N("accept",              K_ACCEPT,              ARGS,    NO_OBJ ),
+  T0N("reject6",             K_REJECT6,             ARGS,    NO_OBJ ),
+  T0N("accept6",             K_ACCEPT6,             ARGS,    NO_OBJ ),
+  T1_START( "router",        K_ROUTER,              GE(5),   NO_OBJ ),
+  T01("ipv6-policy",         K_IPV6_POLICY,         CONCAT_ARGS, NO_OBJ),
+  T1( "signing-key",         K_SIGNING_KEY,         NO_ARGS, NEED_KEY_1024 ),
+  T1( "onion-key",           K_ONION_KEY,           NO_ARGS, NEED_KEY_1024 ),
+  T01("ntor-onion-key",      K_ONION_KEY_NTOR,      GE(1), NO_OBJ ),
+  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
+  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
+  T01("uptime",              K_UPTIME,              GE(1),   NO_OBJ ),
+  T01("fingerprint",         K_FINGERPRINT,     CONCAT_ARGS, NO_OBJ ),
+  T01("hibernating",         K_HIBERNATING,         GE(1),   NO_OBJ ),
+  T01("platform",            K_PLATFORM,        CONCAT_ARGS, NO_OBJ ),
+  T01("proto",               K_PROTO,           CONCAT_ARGS, NO_OBJ ),
+  T01("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
+  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
+  T01("extra-info-digest",   K_EXTRA_INFO_DIGEST,   GE(1),   NO_OBJ ),
+  T01("hidden-service-dir",  K_HIDDEN_SERVICE_DIR,  NO_ARGS, NO_OBJ ),
+  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
+  T01("master-key-ed25519",  K_MASTER_KEY_ED25519,  GE(1),   NO_OBJ ),
+  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
+  T01("onion-key-crosscert", K_ONION_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),
+  T01("ntor-onion-key-crosscert", K_NTOR_ONION_KEY_CROSSCERT,
+                                                    EQ(1),   NEED_OBJ ),
+
+  T01("allow-single-hop-exits",K_ALLOW_SINGLE_HOP_EXITS,    NO_ARGS, NO_OBJ ),
+
+  T01("family",              K_FAMILY,              ARGS,    NO_OBJ ),
+  T01("caches-extra-info",   K_CACHES_EXTRA_INFO,   NO_ARGS, NO_OBJ ),
+  T0N("or-address",          K_OR_ADDRESS,          GE(1),   NO_OBJ ),
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T1( "bandwidth",           K_BANDWIDTH,           GE(3),   NO_OBJ ),
+  A01("@purpose",            A_PURPOSE,             GE(1),   NO_OBJ ),
+  T01("tunnelled-dir-server",K_DIR_TUNNELLED,       NO_ARGS, NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in extra-info documents. */
+static token_rule_t extrainfo_token_table[] = {
+  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
+  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
+  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
+  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
+  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
+  T01("dirreq-stats-end",    K_DIRREQ_END,          ARGS,    NO_OBJ ),
+  T01("dirreq-v2-ips",       K_DIRREQ_V2_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-ips",       K_DIRREQ_V3_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-reqs",      K_DIRREQ_V2_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-reqs",      K_DIRREQ_V3_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-share",     K_DIRREQ_V2_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-share",     K_DIRREQ_V3_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v2-resp",      K_DIRREQ_V2_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-resp",      K_DIRREQ_V3_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN,     ARGS,    NO_OBJ ),
+  T01("entry-stats-end",     K_ENTRY_END,           ARGS,    NO_OBJ ),
+  T01("entry-ips",           K_ENTRY_IPS,           ARGS,    NO_OBJ ),
+  T01("cell-stats-end",      K_CELL_END,            ARGS,    NO_OBJ ),
+  T01("cell-processed-cells", K_CELL_PROCESSED,     ARGS,    NO_OBJ ),
+  T01("cell-queued-cells",   K_CELL_QUEUED,         ARGS,    NO_OBJ ),
+  T01("cell-time-in-queue",  K_CELL_TIME,           ARGS,    NO_OBJ ),
+  T01("cell-circuits-per-decile", K_CELL_CIRCS,     ARGS,    NO_OBJ ),
+  T01("exit-stats-end",      K_EXIT_END,            ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-written", K_EXIT_WRITTEN,     ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-read", K_EXIT_READ,           ARGS,    NO_OBJ ),
+  T01("exit-streams-opened", K_EXIT_OPENED,         ARGS,    NO_OBJ ),
+
+  T1_START( "extra-info",          K_EXTRA_INFO,          GE(2),   NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the body part of v3 networkstatus
+ * documents. */
+static token_rule_t rtrstatus_token_table[] = {
+  T01("p",                   K_P,               CONCAT_ARGS, NO_OBJ ),
+  T1( "r",                   K_R,                   GE(7),   NO_OBJ ),
+  T0N("a",                   K_A,                   GE(1),   NO_OBJ ),
+  T1( "s",                   K_S,                   ARGS,    NO_OBJ ),
+  T01("v",                   K_V,               CONCAT_ARGS, NO_OBJ ),
+  T01("w",                   K_W,                   ARGS,    NO_OBJ ),
+  T0N("m",                   K_M,               CONCAT_ARGS, NO_OBJ ),
+  T0N("id",                  K_ID,                  GE(2),   NO_OBJ ),
+  T01("pr",                  K_PROTO,           CONCAT_ARGS, NO_OBJ ),
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  END_OF_TABLE
+};
+
+/** List of tokens common to V3 authority certificates and V3 consensuses. */
+#define CERTIFICATE_MEMBERS                                                  \
+  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,           \
+                                                     GE(1),       NO_OBJ ),  \
+  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),   \
+  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),   \
+  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),\
+  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,                       \
+                                                     NO_ARGS,     NEED_OBJ), \
+  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ),
+
+/** List of tokens recognized in V3 authority certificates. */
+static token_rule_t dir_key_certificate_table[] = {
+  CERTIFICATE_MEMBERS
+  T1("fingerprint",      K_FINGERPRINT,              CONCAT_ARGS, NO_OBJ ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in rendezvous service descriptors */
+static token_rule_t desc_token_table[] = {
+  T1_START("rendezvous-service-descriptor", R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+           EQ(1), NO_OBJ),
+  T1("version", R_VERSION, EQ(1), NO_OBJ),
+  T1("permanent-key", R_PERMANENT_KEY, NO_ARGS, NEED_KEY_1024),
+  T1("secret-id-part", R_SECRET_ID_PART, EQ(1), NO_OBJ),
+  T1("publication-time", R_PUBLICATION_TIME, CONCAT_ARGS, NO_OBJ),
+  T1("protocol-versions", R_PROTOCOL_VERSIONS, EQ(1), NO_OBJ),
+  T01("introduction-points", R_INTRODUCTION_POINTS, NO_ARGS, NEED_OBJ),
+  T1_END("signature", R_SIGNATURE, NO_ARGS, NEED_OBJ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the (encrypted) list of introduction points of
+ * rendezvous service descriptors */
+static token_rule_t ipo_token_table[] = {
+  T1_START("introduction-point", R_IPO_IDENTIFIER, EQ(1), NO_OBJ),
+  T1("ip-address", R_IPO_IP_ADDRESS, EQ(1), NO_OBJ),
+  T1("onion-port", R_IPO_ONION_PORT, EQ(1), NO_OBJ),
+  T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024),
+  T1("service-key", R_IPO_SERVICE_KEY, NO_ARGS, NEED_KEY_1024),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the (possibly encrypted) list of introduction
+ * points of rendezvous service descriptors */
+static token_rule_t client_keys_token_table[] = {
+  T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+  T1("descriptor-cookie", C_DESCRIPTOR_COOKIE, EQ(1), NO_OBJ),
+  T01("client-key", C_CLIENT_KEY, NO_ARGS, NEED_SKEY_1024),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in V3 networkstatus votes. */
+static token_rule_t networkstatus_token_table[] = {
+  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
+                                                   GE(1),       NO_OBJ ),
+  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
+  T1("published",              K_PUBLISHED,        CONCAT_ARGS, NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
+  T1("known-flags",            K_KNOWN_FLAGS,      ARGS,        NO_OBJ ),
+  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
+  T( "fingerprint",            K_FINGERPRINT,      CONCAT_ARGS, NO_OBJ ),
+  T01("signing-ed25519",       K_SIGNING_CERT_ED,  NO_ARGS ,    NEED_OBJ ),
+  T01("shared-rand-participate",K_SR_FLAG,         NO_ARGS,     NO_OBJ ),
+  T0N("shared-rand-commit",    K_COMMIT,           GE(3),       NO_OBJ ),
+  T01("shared-rand-previous-value", K_PREVIOUS_SRV,EQ(2),       NO_OBJ ),
+  T01("shared-rand-current-value",  K_CURRENT_SRV, EQ(2),       NO_OBJ ),
+  T0N("package",               K_PACKAGE,          CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+
+  CERTIFICATE_MEMBERS
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T1( "contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T1( "dir-source",          K_DIR_SOURCE,      GE(6),       NO_OBJ ),
+  T01("legacy-dir-key",      K_LEGACY_DIR_KEY,  GE(1),       NO_OBJ ),
+  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
+  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T1( "consensus-methods",   K_CONSENSUS_METHODS, GE(1),     NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in V3 networkstatus consensuses. */
+static token_rule_t networkstatus_consensus_token_table[] = {
+  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
+                                                   GE(1),       NO_OBJ ),
+  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+
+  T1N("dir-source",          K_DIR_SOURCE,          GE(6),   NO_OBJ ),
+  T1N("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T1N("vote-digest",         K_VOTE_DIGEST,         GE(1),   NO_OBJ ),
+
+  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
+
+  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("consensus-method",    K_CONSENSUS_METHOD,    EQ(1),   NO_OBJ),
+  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
+
+  T01("shared-rand-previous-value", K_PREVIOUS_SRV, EQ(2),   NO_OBJ ),
+  T01("shared-rand-current-value",  K_CURRENT_SRV,  EQ(2),   NO_OBJ ),
+
+  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the footer of v1 directory footers. */
+static token_rule_t networkstatus_vote_footer_token_table[] = {
+  T01("directory-footer",    K_DIRECTORY_FOOTER,    NO_ARGS,   NO_OBJ ),
+  T01("bandwidth-weights",   K_BW_WEIGHTS,          ARGS,      NO_OBJ ),
+  T(  "directory-signature", K_DIRECTORY_SIGNATURE, GE(2),     NEED_OBJ ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in detached networkstatus signature documents. */
+static token_rule_t networkstatus_detached_signature_token_table[] = {
+  T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1),       NO_OBJ ),
+  T("additional-digest",       K_ADDITIONAL_DIGEST,GE(3),       NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T("additional-signature",  K_ADDITIONAL_SIGNATURE, GE(4),   NEED_OBJ ),
+  T1N("directory-signature", K_DIRECTORY_SIGNATURE,  GE(2),   NEED_OBJ ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in microdescriptors */
+static token_rule_t microdesc_token_table[] = {
+  T1_START("onion-key",        K_ONION_KEY,        NO_ARGS,     NEED_KEY_1024),
+  T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
+  T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
+  T0N("a",                     K_A,                GE(1),       NO_OBJ ),
+  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
+  T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
+  A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
+  END_OF_TABLE
+};
+
+#undef T
+
+/* static function prototypes */
+static int router_add_exit_policy(routerinfo_t *router,directory_token_t *tok);
+static addr_policy_t *router_parse_addr_policy(directory_token_t *tok,
+                                               unsigned fmt_flags);
+static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
+
+static int router_get_hash_impl_helper(const char *s, size_t s_len,
+                            const char *start_str,
+                            const char *end_str, char end_c,
+                            int log_severity,
+                            const char **start_out, const char **end_out);
+static int router_get_hash_impl(const char *s, size_t s_len, char *digest,
+                                const char *start_str, const char *end_str,
+                                char end_char,
+                                digest_algorithm_t alg);
+static int router_get_hashes_impl(const char *s, size_t s_len,
+                                  common_digests_t *digests,
+                                  const char *start_str, const char *end_str,
+                                  char end_char);
+static smartlist_t *find_all_exitpolicy(smartlist_t *s);
+
+#define CST_NO_CHECK_OBJTYPE  (1<<0)
+static int check_signature_token(const char *digest,
+                                 ssize_t digest_len,
+                                 directory_token_t *tok,
+                                 crypto_pk_t *pkey,
+                                 int flags,
+                                 const char *doctype);
+
+#undef DEBUG_AREA_ALLOC
+
+#ifdef DEBUG_AREA_ALLOC
+#define DUMP_AREA(a,name) STMT_BEGIN                              \
+  size_t alloc=0, used=0;                                         \
+  memarea_get_stats((a),&alloc,&used);                            \
+  log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
+            name, (unsigned long)alloc, (unsigned long)used);     \
+  STMT_END
+#else /* !(defined(DEBUG_AREA_ALLOC)) */
+#define DUMP_AREA(a,name) STMT_NIL
+#endif /* defined(DEBUG_AREA_ALLOC) */
+
+/* Dump mechanism for unparseable descriptors */
+
+/** List of dumped descriptors for FIFO cleanup purposes */
+STATIC smartlist_t *descs_dumped = NULL;
+/** Total size of dumped descriptors for FIFO cleanup */
+STATIC uint64_t len_descs_dumped = 0;
+/** Directory to stash dumps in */
+static int have_dump_desc_dir = 0;
+static int problem_with_dump_desc_dir = 0;
+
+#define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs"
+#define DESC_DUMP_BASE_FILENAME "unparseable-desc"
+
+/** Find the dump directory and check if we'll be able to create it */
+static void
+dump_desc_init(void)
+{
+  char *dump_desc_dir;
+
+  dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+  /*
+   * We just check for it, don't create it at this point; we'll
+   * create it when we need it if it isn't already there.
+   */
+  if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) {
+    /* Error, log and flag it as having a problem */
+    log_notice(LD_DIR,
+               "Doesn't look like we'll be able to create descriptor dump "
+               "directory %s; dumps will be disabled.",
+               dump_desc_dir);
+    problem_with_dump_desc_dir = 1;
+    tor_free(dump_desc_dir);
+    return;
+  }
+
+  /* Check if it exists */
+  switch (file_status(dump_desc_dir)) {
+    case FN_DIR:
+      /* We already have a directory */
+      have_dump_desc_dir = 1;
+      break;
+    case FN_NOENT:
+      /* Nothing, we'll need to create it later */
+      have_dump_desc_dir = 0;
+      break;
+    case FN_ERROR:
+      /* Log and flag having a problem */
+      log_notice(LD_DIR,
+                 "Couldn't check whether descriptor dump directory %s already"
+                 " exists: %s",
+                 dump_desc_dir, strerror(errno));
+      problem_with_dump_desc_dir = 1;
+      break;
+    case FN_FILE:
+    case FN_EMPTY:
+    default:
+      /* Something else was here! */
+      log_notice(LD_DIR,
+                 "Descriptor dump directory %s already exists and isn't a "
+                 "directory",
+                 dump_desc_dir);
+      problem_with_dump_desc_dir = 1;
+  }
+
+  if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+    dump_desc_populate_fifo_from_directory(dump_desc_dir);
+  }
+
+  tor_free(dump_desc_dir);
+}
+
+/** Create the dump directory if needed and possible */
+static void
+dump_desc_create_dir(void)
+{
+  char *dump_desc_dir;
+
+  /* If the problem flag is set, skip it */
+  if (problem_with_dump_desc_dir) return;
+
+  /* Do we need it? */
+  if (!have_dump_desc_dir) {
+    dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+    if (check_private_dir(dump_desc_dir, CPD_CREATE,
+                          get_options()->User) < 0) {
+      log_notice(LD_DIR,
+                 "Failed to create descriptor dump directory %s",
+                 dump_desc_dir);
+      problem_with_dump_desc_dir = 1;
+    }
+
+    /* Okay, we created it */
+    have_dump_desc_dir = 1;
+
+    tor_free(dump_desc_dir);
+  }
+}
+
+/** Dump desc FIFO/cleanup; take ownership of the given filename, add it to
+ * the FIFO, and clean up the oldest entries to the extent they exceed the
+ * configured cap.  If any old entries with a matching hash existed, they
+ * just got overwritten right before this was called and we should adjust
+ * the total size counter without deleting them.
+ */
+static void
+dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256,
+                             size_t len)
+{
+  dumped_desc_t *ent = NULL, *tmp;
+  uint64_t max_len;
+
+  tor_assert(filename != NULL);
+  tor_assert(digest_sha256 != NULL);
+
+  if (descs_dumped == NULL) {
+    /* We better have no length, then */
+    tor_assert(len_descs_dumped == 0);
+    /* Make a smartlist */
+    descs_dumped = smartlist_new();
+  }
+
+  /* Make a new entry to put this one in */
+  ent = tor_malloc_zero(sizeof(*ent));
+  ent->filename = filename;
+  ent->len = len;
+  ent->when = time(NULL);
+  memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN);
+
+  /* Do we need to do some cleanup? */
+  max_len = get_options()->MaxUnparseableDescSizeToLog;
+  /* Iterate over the list until we've freed enough space */
+  while (len > max_len - len_descs_dumped &&
+         smartlist_len(descs_dumped) > 0) {
+    /* Get the oldest thing on the list */
+    tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0));
+
+    /*
+     * Check if it matches the filename we just added, so we don't delete
+     * something we just emitted if we get repeated identical descriptors.
+     */
+    if (strcmp(tmp->filename, filename) != 0) {
+      /* Delete it and adjust the length counter */
+      tor_unlink(tmp->filename);
+      tor_assert(len_descs_dumped >= tmp->len);
+      len_descs_dumped -= tmp->len;
+      log_info(LD_DIR,
+               "Deleting old unparseable descriptor dump %s due to "
+               "space limits",
+               tmp->filename);
+    } else {
+      /*
+       * Don't delete, but do adjust the counter since we will bump it
+       * later
+       */
+      tor_assert(len_descs_dumped >= tmp->len);
+      len_descs_dumped -= tmp->len;
+      log_info(LD_DIR,
+               "Replacing old descriptor dump %s with new identical one",
+               tmp->filename);
+    }
+
+    /* Free it and remove it from the list */
+    smartlist_del_keeporder(descs_dumped, 0);
+    tor_free(tmp->filename);
+    tor_free(tmp);
+  }
+
+  /* Append our entry to the end of the list and bump the counter */
+  smartlist_add(descs_dumped, ent);
+  len_descs_dumped += len;
+}
+
+/** Check if we already have a descriptor for this hash and move it to the
+ * head of the queue if so.  Return 1 if one existed and 0 otherwise.
+ */
+static int
+dump_desc_fifo_bump_hash(const uint8_t *digest_sha256)
+{
+  dumped_desc_t *match = NULL;
+
+  tor_assert(digest_sha256);
+
+  if (descs_dumped) {
+    /* Find a match if one exists */
+    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+      if (ent &&
+          tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) {
+        /*
+         * Save a pointer to the match and remove it from its current
+         * position.
+         */
+        match = ent;
+        SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent);
+        break;
+      }
+    } SMARTLIST_FOREACH_END(ent);
+
+    if (match) {
+      /* Update the timestamp */
+      match->when = time(NULL);
+      /* Add it back at the end of the list */
+      smartlist_add(descs_dumped, match);
+
+      /* Indicate we found one */
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+/** Clean up on exit; just memory, leave the dumps behind
+ */
+STATIC void
+dump_desc_fifo_cleanup(void)
+{
+  if (descs_dumped) {
+    /* Free each descriptor */
+    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+      tor_assert(ent);
+      tor_free(ent->filename);
+      tor_free(ent);
+    } SMARTLIST_FOREACH_END(ent);
+    /* Free the list */
+    smartlist_free(descs_dumped);
+    descs_dumped = NULL;
+    len_descs_dumped = 0;
+  }
+}
+
+/** Handle one file for dump_desc_populate_fifo_from_directory(); make sure
+ * the filename is sensibly formed and matches the file content, and either
+ * return a dumped_desc_t for it or remove the file and return NULL.
+ */
+MOCK_IMPL(STATIC dumped_desc_t *,
+dump_desc_populate_one_file, (const char *dirname, const char *f))
+{
+  dumped_desc_t *ent = NULL;
+  char *path = NULL, *desc = NULL;
+  const char *digest_str;
+  char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN];
+  /* Expected prefix before digest in filenames */
+  const char *f_pfx = DESC_DUMP_BASE_FILENAME ".";
+  /*
+   * Stat while reading; this is important in case the file
+   * contains a NUL character.
+   */
+  struct stat st;
+
+  /* Sanity-check args */
+  tor_assert(dirname != NULL);
+  tor_assert(f != NULL);
+
+  /* Form the full path */
+  tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f);
+
+  /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */
+
+  if (!strcmpstart(f, f_pfx)) {
+    /* It matches the form, but is the digest parseable as such? */
+    digest_str = f + strlen(f_pfx);
+    if (base16_decode(digest, DIGEST256_LEN,
+                      digest_str, strlen(digest_str)) != DIGEST256_LEN) {
+      /* We failed to decode it */
+      digest_str = NULL;
+    }
+  } else {
+    /* No match */
+    digest_str = NULL;
+  }
+
+  if (!digest_str) {
+    /* We couldn't get a sensible digest */
+    log_notice(LD_DIR,
+               "Removing unrecognized filename %s from unparseable "
+               "descriptors directory", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /*
+   * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and
+   * we've decoded the digest.  Next, check that we can read it and the
+   * content matches this digest.  We are relying on the fact that if the
+   * file contains a '\0', read_file_to_str() will allocate space for and
+   * read the entire file and return the correct size in st.
+   */
+  desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st);
+  if (!desc) {
+    /* We couldn't read it */
+    log_notice(LD_DIR,
+               "Failed to read %s from unparseable descriptors directory; "
+               "attempting to remove it.", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+#if SIZE_MAX > UINT64_MAX
+  if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) {
+    /* LCOV_EXCL_START
+     * Should be impossible since RFTS above should have failed to read the
+     * huge file into RAM. */
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+#endif /* SIZE_MAX > UINT64_MAX */
+  if (BUG(st.st_size < 0)) {
+    /* LCOV_EXCL_START
+     * Should be impossible, since the OS isn't supposed to be b0rken. */
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+  /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */
+
+  /*
+   * We got one; now compute its digest and check that it matches the
+   * filename.
+   */
+  if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
+                       DIGEST_SHA256) < 0) {
+    /* Weird, but okay */
+    log_info(LD_DIR,
+             "Unable to hash content of %s from unparseable descriptors "
+             "directory", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /* Compare the digests */
+  if (tor_memneq(digest, content_digest, DIGEST256_LEN)) {
+    /* No match */
+    log_info(LD_DIR,
+             "Hash of %s from unparseable descriptors directory didn't "
+             "match its filename; removing it", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /* Okay, it's a match, we should prepare ent */
+  ent = tor_malloc_zero(sizeof(dumped_desc_t));
+  ent->filename = path;
+  memcpy(ent->digest_sha256, digest, DIGEST256_LEN);
+  ent->len = (size_t) st.st_size;
+  ent->when = st.st_mtime;
+  /* Null out path so we don't free it out from under ent */
+  path = NULL;
+
+ done:
+  /* Free allocations if we had them */
+  tor_free(desc);
+  tor_free(path);
+
+  return ent;
+}
+
+/** Sort helper for dump_desc_populate_fifo_from_directory(); compares
+ * the when field of dumped_desc_ts in a smartlist to put the FIFO in
+ * the correct order after reconstructing it from the directory.
+ */
+static int
+dump_desc_compare_fifo_entries(const void **a_v, const void **b_v)
+{
+  const dumped_desc_t **a = (const dumped_desc_t **)a_v;
+  const dumped_desc_t **b = (const dumped_desc_t **)b_v;
+
+  if ((a != NULL) && (*a != NULL)) {
+    if ((b != NULL) && (*b != NULL)) {
+      /* We have sensible dumped_desc_ts to compare */
+      if ((*a)->when < (*b)->when) {
+        return -1;
+      } else if ((*a)->when == (*b)->when) {
+        return 0;
+      } else {
+        return 1;
+      }
+    } else {
+      /*
+       * We shouldn't see this, but what the hell, NULLs precede everythin
+       * else
+       */
+      return 1;
+    }
+  } else {
+    return -1;
+  }
+}
+
+/** Scan the contents of the directory, and update FIFO/counters; this will
+ * consistency-check descriptor dump filenames against hashes of descriptor
+ * dump file content, and remove any inconsistent/unreadable dumps, and then
+ * reconstruct the dump FIFO as closely as possible for the last time the
+ * tor process shut down.  If a previous dump was repeated more than once and
+ * moved ahead in the FIFO, the mtime will not have been updated and the
+ * reconstructed order will be wrong, but will always be a permutation of
+ * the original.
+ */
+STATIC void
+dump_desc_populate_fifo_from_directory(const char *dirname)
+{
+  smartlist_t *files = NULL;
+  dumped_desc_t *ent = NULL;
+
+  tor_assert(dirname != NULL);
+
+  /* Get a list of files */
+  files = tor_listdir(dirname);
+  if (!files) {
+    log_notice(LD_DIR,
+               "Unable to get contents of unparseable descriptor dump "
+               "directory %s",
+               dirname);
+    return;
+  }
+
+  /*
+   * Iterate through the list and decide which files should go in the
+   * FIFO and which should be purged.
+   */
+
+  SMARTLIST_FOREACH_BEGIN(files, char *, f) {
+    /* Try to get a FIFO entry */
+    ent = dump_desc_populate_one_file(dirname, f);
+    if (ent) {
+      /*
+       * We got one; add it to the FIFO.  No need for duplicate checking
+       * here since we just verified the name and digest match.
+       */
+
+      /* Make sure we have a list to add it to */
+      if (!descs_dumped) {
+        descs_dumped = smartlist_new();
+        len_descs_dumped = 0;
+      }
+
+      /* Add it and adjust the counter */
+      smartlist_add(descs_dumped, ent);
+      len_descs_dumped += ent->len;
+    }
+    /*
+     * If we didn't, we will have unlinked the file if necessary and
+     * possible, and emitted a log message about it, so just go on to
+     * the next.
+     */
+  } SMARTLIST_FOREACH_END(f);
+
+  /* Did we get anything? */
+  if (descs_dumped != NULL) {
+    /* Sort the FIFO in order of increasing timestamp */
+    smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries);
+
+    /* Log some stats */
+    log_info(LD_DIR,
+             "Reloaded unparseable descriptor dump FIFO with %d dump(s) "
+             "totaling %"PRIu64 " bytes",
+             smartlist_len(descs_dumped), (len_descs_dumped));
+  }
+
+  /* Free the original list */
+  SMARTLIST_FOREACH(files, char *, f, tor_free(f));
+  smartlist_free(files);
+}
+
+/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of
+ * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
+ * than one descriptor to disk per minute. If there is already such a
+ * file in the data directory, overwrite it. */
+MOCK_IMPL(STATIC void,
+dump_desc,(const char *desc, const char *type))
+{
+  tor_assert(desc);
+  tor_assert(type);
+  size_t len;
+  /* The SHA256 of the string */
+  uint8_t digest_sha256[DIGEST256_LEN];
+  char digest_sha256_hex[HEX_DIGEST256_LEN+1];
+  /* Filename to log it to */
+  char *debugfile, *debugfile_base;
+
+  /* Get the hash for logging purposes anyway */
+  len = strlen(desc);
+  if (crypto_digest256((char *)digest_sha256, desc, len,
+                       DIGEST_SHA256) < 0) {
+    log_info(LD_DIR,
+             "Unable to parse descriptor of type %s, and unable to even hash"
+             " it!", type);
+    goto err;
+  }
+
+  base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex),
+                (const char *)digest_sha256, sizeof(digest_sha256));
+
+  /*
+   * We mention type and hash in the main log; don't clutter up the files
+   * with anything but the exact dump.
+   */
+  tor_asprintf(&debugfile_base,
+               DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex);
+  debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base);
+
+  /*
+   * Check if the sandbox is active or will become active; see comment
+   * below at the log message for why.
+   */
+  if (!(sandbox_is_active() || get_options()->Sandbox)) {
+    if (len <= get_options()->MaxUnparseableDescSizeToLog) {
+      if (!dump_desc_fifo_bump_hash(digest_sha256)) {
+        /* Create the directory if needed */
+        dump_desc_create_dir();
+        /* Make sure we've got it */
+        if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+          /* Write it, and tell the main log about it */
+          write_str_to_file(debugfile, desc, 1);
+          log_info(LD_DIR,
+                   "Unable to parse descriptor of type %s with hash %s and "
+                   "length %lu. See file %s in data directory for details.",
+                   type, digest_sha256_hex, (unsigned long)len,
+                   debugfile_base);
+          dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len);
+          /* Since we handed ownership over, don't free debugfile later */
+          debugfile = NULL;
+        } else {
+          /* Problem with the subdirectory */
+          log_info(LD_DIR,
+                   "Unable to parse descriptor of type %s with hash %s and "
+                   "length %lu. Descriptor not dumped because we had a "
+                   "problem creating the " DESC_DUMP_DATADIR_SUBDIR
+                   " subdirectory",
+                   type, digest_sha256_hex, (unsigned long)len);
+          /* We do have to free debugfile in this case */
+        }
+      } else {
+        /* We already had one with this hash dumped */
+        log_info(LD_DIR,
+                 "Unable to parse descriptor of type %s with hash %s and "
+                 "length %lu. Descriptor not dumped because one with that "
+                 "hash has already been dumped.",
+                 type, digest_sha256_hex, (unsigned long)len);
+        /* We do have to free debugfile in this case */
+      }
+    } else {
+      /* Just log that it happened without dumping */
+      log_info(LD_DIR,
+               "Unable to parse descriptor of type %s with hash %s and "
+               "length %lu. Descriptor not dumped because it exceeds maximum"
+               " log size all by itself.",
+               type, digest_sha256_hex, (unsigned long)len);
+      /* We do have to free debugfile in this case */
+    }
+  } else {
+    /*
+     * Not logging because the sandbox is active and seccomp2 apparently
+     * doesn't have a sensible way to allow filenames according to a pattern
+     * match.  (If we ever figure out how to say "allow writes to /regex/",
+     * remove this checK).
+     */
+    log_info(LD_DIR,
+             "Unable to parse descriptor of type %s with hash %s and "
+             "length %lu. Descriptor not dumped because the sandbox is "
+             "configured",
+             type, digest_sha256_hex, (unsigned long)len);
+  }
+
+  tor_free(debugfile_base);
+  tor_free(debugfile);
+
+ err:
+  return;
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in
+ * <b>s</b>.  Return 0 on success, -1 on failure.
+ */
+int
+router_get_dir_hash(const char *s, char *digest)
+{
+  return router_get_hash_impl(s, strlen(s), digest,
+                              "signed-directory","\ndirectory-signature",'\n',
+                              DIGEST_SHA1);
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
+ * <b>s</b>. Return 0 on success, -1 on failure.
+ */
+int
+router_get_router_hash(const char *s, size_t s_len, char *digest)
+{
+  return router_get_hash_impl(s, s_len, digest,
+                              "router ","\nrouter-signature", '\n',
+                              DIGEST_SHA1);
+}
+
+/** Try to find the start and end of the signed portion of a networkstatus
+ * document in <b>s</b>. On success, set <b>start_out</b> to the first
+ * character of the document, and <b>end_out</b> to a position one after the
+ * final character of the signed document, and return 0.  On failure, return
+ * -1. */
+int
+router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              const char **start_out,
+                                              const char **end_out)
+{
+  return router_get_hash_impl_helper(s, strlen(s),
+                                     "network-status-version",
+                                     "\ndirectory-signature",
+                                     ' ', LOG_INFO,
+                                     start_out, end_out);
+}
+
+/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the
+ * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no
+ * signed portion can be identified.  Return 0 on success, -1 on failure. */
+int
+router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                           const char *s)
+{
+  const char *start, *end;
+  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+    start = s;
+    end = s + strlen(s);
+  }
+  tor_assert(start);
+  tor_assert(end);
+  return crypto_digest256((char*)digest_out, start, end-start,
+                          DIGEST_SHA3_256);
+}
+
+/** Set <b>digests</b> to all the digests of the consensus document in
+ * <b>s</b> */
+int
+router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
+{
+  return router_get_hashes_impl(s,strlen(s),digests,
+                                "network-status-version",
+                                "\ndirectory-signature",
+                                ' ');
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the <b>s_len</b>-byte
+ * extrainfo string at <b>s</b>.  Return 0 on success, -1 on failure. */
+int
+router_get_extrainfo_hash(const char *s, size_t s_len, char *digest)
+{
+  return router_get_hash_impl(s, s_len, digest, "extra-info",
+                              "\nrouter-signature",'\n', DIGEST_SHA1);
+}
+
+/** Helper: used to generate signatures for routers, directories and
+ * network-status objects.  Given a <b>digest_len</b>-byte digest in
+ * <b>digest</b> and a secret <b>private_key</b>, generate an PKCS1-padded
+ * signature, BASE64-encode it, surround it with -----BEGIN/END----- pairs,
+ * and return the new signature on success or NULL on failure.
+ */
+char *
+router_get_dirobj_signature(const char *digest,
+                            size_t digest_len,
+                            const crypto_pk_t *private_key)
+{
+  char *signature;
+  size_t i, keysize;
+  int siglen;
+  char *buf = NULL;
+  size_t buf_len;
+  /* overestimate of BEGIN/END lines total len. */
+#define BEGIN_END_OVERHEAD_LEN 64
+
+  keysize = crypto_pk_keysize(private_key);
+  signature = tor_malloc(keysize);
+  siglen = crypto_pk_private_sign(private_key, signature, keysize,
+                                  digest, digest_len);
+  if (siglen < 0) {
+    log_warn(LD_BUG,"Couldn't sign digest.");
+    goto err;
+  }
+
+  /* The *2 here is a ridiculous overestimate of base-64 overhead. */
+  buf_len = (siglen * 2) + BEGIN_END_OVERHEAD_LEN;
+  buf = tor_malloc(buf_len);
+
+  if (strlcpy(buf, "-----BEGIN SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  i = strlen(buf);
+  if (base64_encode(buf+i, buf_len-i, signature, siglen,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    log_warn(LD_BUG,"couldn't base64-encode signature");
+    goto err;
+  }
+
+  if (strlcat(buf, "-----END SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  tor_free(signature);
+  return buf;
+
+ truncated:
+  log_warn(LD_BUG,"tried to exceed string length.");
+ err:
+  tor_free(signature);
+  tor_free(buf);
+  return NULL;
+}
+
+/** Helper: used to generate signatures for routers, directories and
+ * network-status objects.  Given a digest in <b>digest</b> and a secret
+ * <b>private_key</b>, generate a PKCS1-padded signature, BASE64-encode it,
+ * surround it with -----BEGIN/END----- pairs, and write it to the
+ * <b>buf_len</b>-byte buffer at <b>buf</b>.  Return 0 on success, -1 on
+ * failure.
+ */
+int
+router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
+                               size_t digest_len, crypto_pk_t *private_key)
+{
+  size_t sig_len, s_len;
+  char *sig = router_get_dirobj_signature(digest, digest_len, private_key);
+  if (!sig) {
+    log_warn(LD_BUG, "No signature generated");
+    return -1;
+  }
+  sig_len = strlen(sig);
+  s_len = strlen(buf);
+  if (sig_len + s_len + 1 > buf_len) {
+    log_warn(LD_BUG, "Not enough room for signature");
+    tor_free(sig);
+    return -1;
+  }
+  memcpy(buf+s_len, sig, sig_len+1);
+  tor_free(sig);
+  return 0;
+}
+
+/** Return VS_RECOMMENDED if <b>myversion</b> is contained in
+ * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
+ * entries. Else, return VS_OLD if every member of
+ * <b>versionlist</b> is newer than <b>myversion</b>.  Else, return
+ * VS_NEW_IN_SERIES if there is at least one member of <b>versionlist</b> in
+ * the same series (major.minor.micro) as <b>myversion</b>, but no such member
+ * is newer than <b>myversion.</b>.  Else, return VS_NEW if every member of
+ * <b>versionlist</b> is older than <b>myversion</b>.  Else, return
+ * VS_UNRECOMMENDED.
+ *
+ * (versionlist is a comma-separated list of version strings,
+ * optionally prefixed with "Tor".  Versions that can't be parsed are
+ * ignored.)
+ */
+version_status_t
+tor_version_is_obsolete(const char *myversion, const char *versionlist)
+{
+  tor_version_t mine, other;
+  int found_newer = 0, found_older = 0, found_newer_in_series = 0,
+    found_any_in_series = 0, r, same;
+  version_status_t ret = VS_UNRECOMMENDED;
+  smartlist_t *version_sl;
+
+  log_debug(LD_CONFIG,"Checking whether version '%s' is in '%s'",
+            myversion, versionlist);
+
+  if (tor_version_parse(myversion, &mine)) {
+    log_err(LD_BUG,"I couldn't parse my own version (%s)", myversion);
+    tor_assert(0);
+  }
+  version_sl = smartlist_new();
+  smartlist_split_string(version_sl, versionlist, ",", SPLIT_SKIP_SPACE, 0);
+
+  if (!strlen(versionlist)) { /* no authorities cared or agreed */
+    ret = VS_EMPTY;
+    goto done;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(version_sl, const char *, cp) {
+    if (!strcmpstart(cp, "Tor "))
+      cp += 4;
+
+    if (tor_version_parse(cp, &other)) {
+      /* Couldn't parse other; it can't be a match. */
+    } else {
+      same = tor_version_same_series(&mine, &other);
+      if (same)
+        found_any_in_series = 1;
+      r = tor_version_compare(&mine, &other);
+      if (r==0) {
+        ret = VS_RECOMMENDED;
+        goto done;
+      } else if (r<0) {
+        found_newer = 1;
+        if (same)
+          found_newer_in_series = 1;
+      } else if (r>0) {
+        found_older = 1;
+      }
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  /* We didn't find the listed version. Is it new or old? */
+  if (found_any_in_series && !found_newer_in_series && found_newer) {
+    ret = VS_NEW_IN_SERIES;
+  } else if (found_newer && !found_older) {
+    ret = VS_OLD;
+  } else if (found_older && !found_newer) {
+    ret = VS_NEW;
+  } else {
+    ret = VS_UNRECOMMENDED;
+  }
+
+ done:
+  SMARTLIST_FOREACH(version_sl, char *, version, tor_free(version));
+  smartlist_free(version_sl);
+  return ret;
+}
+
+MOCK_IMPL(STATIC int,
+signed_digest_equals, (const uint8_t *d1, const uint8_t *d2, size_t len))
+{
+  return tor_memeq(d1, d2, len);
+}
+
+/** Check whether the object body of the token in <b>tok</b> has a good
+ * signature for <b>digest</b> using key <b>pkey</b>.
+ * If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
+ * the object type of the signature object. Use <b>doctype</b> as the type of
+ * the document when generating log messages.  Return 0 on success, negative
+ * on failure.
+ */
+static int
+check_signature_token(const char *digest,
+                      ssize_t digest_len,
+                      directory_token_t *tok,
+                      crypto_pk_t *pkey,
+                      int flags,
+                      const char *doctype)
+{
+  char *signed_digest;
+  size_t keysize;
+  const int check_objtype = ! (flags & CST_NO_CHECK_OBJTYPE);
+
+  tor_assert(pkey);
+  tor_assert(tok);
+  tor_assert(digest);
+  tor_assert(doctype);
+
+  if (check_objtype) {
+    if (strcmp(tok->object_type, "SIGNATURE")) {
+      log_warn(LD_DIR, "Bad object type on %s signature", doctype);
+      return -1;
+    }
+  }
+
+  keysize = crypto_pk_keysize(pkey);
+  signed_digest = tor_malloc(keysize);
+  if (crypto_pk_public_checksig(pkey, signed_digest, keysize,
+                                tok->object_body, tok->object_size)
+      < digest_len) {
+    log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype);
+    tor_free(signed_digest);
+    return -1;
+  }
+  //  log_debug(LD_DIR,"Signed %s hash starts %s", doctype,
+  //            hex_str(signed_digest,4));
+  if (! signed_digest_equals((const uint8_t *)digest,
+                             (const uint8_t *)signed_digest, digest_len)) {
+    log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype);
+    tor_free(signed_digest);
+    return -1;
+  }
+  tor_free(signed_digest);
+  return 0;
+}
+
+/** Helper: move *<b>s_ptr</b> ahead to the next router, the next extra-info,
+ * or to the first of the annotations proceeding the next router or
+ * extra-info---whichever comes first.  Set <b>is_extrainfo_out</b> to true if
+ * we found an extrainfo, or false if found a router. Do not scan beyond
+ * <b>eos</b>.  Return -1 if we found nothing; 0 if we found something. */
+static int
+find_start_of_next_router_or_extrainfo(const char **s_ptr,
+                                       const char *eos,
+                                       int *is_extrainfo_out)
+{
+  const char *annotations = NULL;
+  const char *s = *s_ptr;
+
+  s = eat_whitespace_eos(s, eos);
+
+  while (s < eos-32) {  /* 32 gives enough room for a the first keyword. */
+    /* We're at the start of a line. */
+    tor_assert(*s != '\n');
+
+    if (*s == '@' && !annotations) {
+      annotations = s;
+    } else if (*s == 'r' && !strcmpstart(s, "router ")) {
+      *s_ptr = annotations ? annotations : s;
+      *is_extrainfo_out = 0;
+      return 0;
+    } else if (*s == 'e' && !strcmpstart(s, "extra-info ")) {
+      *s_ptr = annotations ? annotations : s;
+      *is_extrainfo_out = 1;
+      return 0;
+    }
+
+    if (!(s = memchr(s+1, '\n', eos-(s+1))))
+      break;
+    s = eat_whitespace_eos(s, eos);
+  }
+  return -1;
+}
+
+/** Given a string *<b>s</b> containing a concatenated sequence of router
+ * descriptors (or extra-info documents if <b>is_extrainfo</b> is set), parses
+ * them and stores the result in <b>dest</b>.  All routers are marked running
+ * and valid.  Advances *s to a point immediately following the last router
+ * entry.  Ignore any trailing router entries that are not complete.
+ *
+ * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
+ * descriptor in the signed_descriptor_body field of each routerinfo_t.  If it
+ * isn't SAVED_NOWHERE, remember the offset of each descriptor.
+ *
+ * Returns 0 on success and -1 on failure.  Adds a digest to
+ * <b>invalid_digests_out</b> for every entry that was unparseable or
+ * invalid. (This may cause duplicate entries.)
+ */
+int
+router_parse_list_from_string(const char **s, const char *eos,
+                              smartlist_t *dest,
+                              saved_location_t saved_location,
+                              int want_extrainfo,
+                              int allow_annotations,
+                              const char *prepend_annotations,
+                              smartlist_t *invalid_digests_out)
+{
+  routerinfo_t *router;
+  extrainfo_t *extrainfo;
+  signed_descriptor_t *signed_desc = NULL;
+  void *elt;
+  const char *end, *start;
+  int have_extrainfo;
+
+  tor_assert(s);
+  tor_assert(*s);
+  tor_assert(dest);
+
+  start = *s;
+  if (!eos)
+    eos = *s + strlen(*s);
+
+  tor_assert(eos >= *s);
+
+  while (1) {
+    char raw_digest[DIGEST_LEN];
+    int have_raw_digest = 0;
+    int dl_again = 0;
+    if (find_start_of_next_router_or_extrainfo(s, eos, &have_extrainfo) < 0)
+      break;
+
+    end = tor_memstr(*s, eos-*s, "\nrouter-signature");
+    if (end)
+      end = tor_memstr(end, eos-end, "\n-----END SIGNATURE-----\n");
+    if (end)
+      end += strlen("\n-----END SIGNATURE-----\n");
+
+    if (!end)
+      break;
+
+    elt = NULL;
+
+    if (have_extrainfo && want_extrainfo) {
+      routerlist_t *rl = router_get_routerlist();
+      have_raw_digest = router_get_extrainfo_hash(*s, end-*s, raw_digest) == 0;
+      extrainfo = extrainfo_parse_entry_from_string(*s, end,
+                                       saved_location != SAVED_IN_CACHE,
+                                       rl->identity_map, &dl_again);
+      if (extrainfo) {
+        signed_desc = &extrainfo->cache_info;
+        elt = extrainfo;
+      }
+    } else if (!have_extrainfo && !want_extrainfo) {
+      have_raw_digest = router_get_router_hash(*s, end-*s, raw_digest) == 0;
+      router = router_parse_entry_from_string(*s, end,
+                                              saved_location != SAVED_IN_CACHE,
+                                              allow_annotations,
+                                              prepend_annotations, &dl_again);
+      if (router) {
+        log_debug(LD_DIR, "Read router '%s', purpose '%s'",
+                  router_describe(router),
+                  router_purpose_to_string(router->purpose));
+        signed_desc = &router->cache_info;
+        elt = router;
+      }
+    }
+    if (! elt && ! dl_again && have_raw_digest && invalid_digests_out) {
+      smartlist_add(invalid_digests_out, tor_memdup(raw_digest, DIGEST_LEN));
+    }
+    if (!elt) {
+      *s = end;
+      continue;
+    }
+    if (saved_location != SAVED_NOWHERE) {
+      tor_assert(signed_desc);
+      signed_desc->saved_location = saved_location;
+      signed_desc->saved_offset = *s - start;
+    }
+    *s = end;
+    smartlist_add(dest, elt);
+  }
+
+  return 0;
+}
+
+/* For debugging: define to count every descriptor digest we've seen so we
+ * know if we need to try harder to avoid duplicate verifies. */
+#undef COUNT_DISTINCT_DIGESTS
+
+#ifdef COUNT_DISTINCT_DIGESTS
+static digestmap_t *verified_digests = NULL;
+#endif
+
+/** Log the total count of the number of distinct router digests we've ever
+ * verified.  When compared to the number of times we've verified routerdesc
+ * signatures <i>in toto</i>, this will tell us if we're doing too much
+ * multiple-verification. */
+void
+dump_distinct_digest_count(int severity)
+{
+#ifdef COUNT_DISTINCT_DIGESTS
+  if (!verified_digests)
+    verified_digests = digestmap_new();
+  tor_log(severity, LD_GENERAL, "%d *distinct* router digests verified",
+      digestmap_size(verified_digests));
+#else /* !(defined(COUNT_DISTINCT_DIGESTS)) */
+  (void)severity; /* suppress "unused parameter" warning */
+#endif /* defined(COUNT_DISTINCT_DIGESTS) */
+}
+
+/** Try to find an IPv6 OR port in <b>list</b> of directory_token_t's
+ * with at least one argument (use GE(1) in setup). If found, store
+ * address and port number to <b>addr_out</b> and
+ * <b>port_out</b>. Return number of OR ports found. */
+static int
+find_single_ipv6_orport(const smartlist_t *list,
+                        tor_addr_t *addr_out,
+                        uint16_t *port_out)
+{
+  int ret = 0;
+  tor_assert(list != NULL);
+  tor_assert(addr_out != NULL);
+  tor_assert(port_out != NULL);
+
+  SMARTLIST_FOREACH_BEGIN(list, directory_token_t *, t) {
+    tor_addr_t a;
+    maskbits_t bits;
+    uint16_t port_min, port_max;
+    tor_assert(t->n_args >= 1);
+    /* XXXX Prop186 the full spec allows much more than this. */
+    if (tor_addr_parse_mask_ports(t->args[0], 0,
+                                  &a, &bits, &port_min,
+                                  &port_max) == AF_INET6 &&
+        bits == 128 &&
+        port_min == port_max) {
+      /* Okay, this is one we can understand. Use it and ignore
+         any potential more addresses in list. */
+      tor_addr_copy(addr_out, &a);
+      *port_out = port_min;
+      ret = 1;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(t);
+
+  return ret;
+}
+
+/** Helper function: reads a single router entry from *<b>s</b> ...
+ * *<b>end</b>.  Mallocs a new router and returns it if all goes well, else
+ * returns NULL.  If <b>cache_copy</b> is true, duplicate the contents of
+ * s through end into the signed_descriptor_body of the resulting
+ * routerinfo_t.
+ *
+ * If <b>end</b> is NULL, <b>s</b> must be properly NUL-terminated.
+ *
+ * If <b>allow_annotations</b>, it's okay to encounter annotations in <b>s</b>
+ * before the router; if it's false, reject the router if it's annotated.  If
+ * <b>prepend_annotations</b> is set, it should contain some annotations:
+ * append them to the front of the router before parsing it, and keep them
+ * around when caching the router.
+ *
+ * Only one of allow_annotations and prepend_annotations may be set.
+ *
+ * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
+ * if it's okay to try to download a descriptor with this same digest again,
+ * and 0 if it isn't.  (It might not be okay to download it again if part of
+ * the part covered by the digest is invalid.)
+ */
+routerinfo_t *
+router_parse_entry_from_string(const char *s, const char *end,
+                               int cache_copy, int allow_annotations,
+                               const char *prepend_annotations,
+                               int *can_dl_again_out)
+{
+  routerinfo_t *router = NULL;
+  char digest[128];
+  smartlist_t *tokens = NULL, *exit_policy_tokens = NULL;
+  directory_token_t *tok;
+  struct in_addr in;
+  const char *start_of_annotations, *cp, *s_dup = s;
+  size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0;
+  int ok = 1;
+  memarea_t *area = NULL;
+  tor_cert_t *ntor_cc_cert = NULL;
+  /* Do not set this to '1' until we have parsed everything that we intend to
+   * parse that's covered by the hash. */
+  int can_dl_again = 0;
+
+  tor_assert(!allow_annotations || !prepend_annotations);
+
+  if (!end) {
+    end = s + strlen(s);
+  }
+
+  /* point 'end' to a point immediately after the final newline. */
+  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
+    --end;
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (prepend_annotations) {
+    if (tokenize_string(area,prepend_annotations,NULL,tokens,
+                        routerdesc_token_table,TS_NOCHECK)) {
+      log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
+      goto err;
+    }
+  }
+
+  start_of_annotations = s;
+  cp = tor_memstr(s, end-s, "\nrouter ");
+  if (!cp) {
+    if (end-s < 7 || strcmpstart(s, "router ")) {
+      log_warn(LD_DIR, "No router keyword found.");
+      goto err;
+    }
+  } else {
+    s = cp+1;
+  }
+
+  if (start_of_annotations != s) { /* We have annotations */
+    if (allow_annotations) {
+      if (tokenize_string(area,start_of_annotations,s,tokens,
+                          routerdesc_token_table,TS_NOCHECK)) {
+        log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
+        goto err;
+      }
+    } else {
+      log_warn(LD_DIR, "Found unexpected annotations on router descriptor not "
+               "loaded from disk.  Dropping it.");
+      goto err;
+    }
+  }
+
+  if (router_get_router_hash(s, end - s, digest) < 0) {
+    log_warn(LD_DIR, "Couldn't compute router hash.");
+    goto err;
+  }
+  {
+    int flags = 0;
+    if (allow_annotations)
+      flags |= TS_ANNOTATIONS_OK;
+    if (prepend_annotations)
+      flags |= TS_ANNOTATIONS_OK|TS_NO_NEW_ANNOTATIONS;
+
+    if (tokenize_string(area,s,end,tokens,routerdesc_token_table, flags)) {
+      log_warn(LD_DIR, "Error tokenizing router descriptor.");
+      goto err;
+    }
+  }
+
+  if (smartlist_len(tokens) < 2) {
+    log_warn(LD_DIR, "Impossibly short router descriptor.");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER);
+  const int router_token_pos = smartlist_pos(tokens, tok);
+  tor_assert(tok->n_args >= 5);
+
+  router = tor_malloc_zero(sizeof(routerinfo_t));
+  router->cert_expiration_time = TIME_MAX;
+  router->cache_info.routerlist_index = -1;
+  router->cache_info.annotations_len = s-start_of_annotations + prepend_len;
+  router->cache_info.signed_descriptor_len = end-s;
+  if (cache_copy) {
+    size_t len = router->cache_info.signed_descriptor_len +
+                 router->cache_info.annotations_len;
+    char *signed_body =
+      router->cache_info.signed_descriptor_body = tor_malloc(len+1);
+    if (prepend_annotations) {
+      memcpy(signed_body, prepend_annotations, prepend_len);
+      signed_body += prepend_len;
+    }
+    /* This assertion will always succeed.
+     * len == signed_desc_len + annotations_len
+     *     == end-s + s-start_of_annotations + prepend_len
+     *     == end-start_of_annotations + prepend_len
+     * We already wrote prepend_len bytes into the buffer; now we're
+     * writing end-start_of_annotations -NM. */
+    tor_assert(signed_body+(end-start_of_annotations) ==
+               router->cache_info.signed_descriptor_body+len);
+    memcpy(signed_body, start_of_annotations, end-start_of_annotations);
+    router->cache_info.signed_descriptor_body[len] = '\0';
+    tor_assert(strlen(router->cache_info.signed_descriptor_body) == len);
+  }
+  memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+
+  router->nickname = tor_strdup(tok->args[0]);
+  if (!is_legal_nickname(router->nickname)) {
+    log_warn(LD_DIR,"Router nickname is invalid");
+    goto err;
+  }
+  if (!tor_inet_aton(tok->args[1], &in)) {
+    log_warn(LD_DIR,"Router address is not an IP address.");
+    goto err;
+  }
+  router->addr = ntohl(in.s_addr);
+
+  router->or_port =
+    (uint16_t) tor_parse_long(tok->args[2],10,0,65535,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR,"Invalid OR port %s", escaped(tok->args[2]));
+    goto err;
+  }
+  router->dir_port =
+    (uint16_t) tor_parse_long(tok->args[4],10,0,65535,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR,"Invalid dir port %s", escaped(tok->args[4]));
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_BANDWIDTH);
+  tor_assert(tok->n_args >= 3);
+  router->bandwidthrate = (int)
+    tor_parse_long(tok->args[0],10,1,INT_MAX,&ok,NULL);
+
+  if (!ok) {
+    log_warn(LD_DIR, "bandwidthrate %s unreadable or 0. Failing.",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  router->bandwidthburst =
+    (int) tor_parse_long(tok->args[1],10,0,INT_MAX,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid bandwidthburst %s", escaped(tok->args[1]));
+    goto err;
+  }
+  router->bandwidthcapacity = (int)
+    tor_parse_long(tok->args[2],10,0,INT_MAX,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid bandwidthcapacity %s", escaped(tok->args[1]));
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, A_PURPOSE))) {
+    tor_assert(tok->n_args);
+    router->purpose = router_purpose_from_string(tok->args[0]);
+  } else {
+    router->purpose = ROUTER_PURPOSE_GENERAL;
+  }
+  router->cache_info.send_unencrypted =
+    (router->purpose == ROUTER_PURPOSE_GENERAL) ? 1 : 0;
+
+  if ((tok = find_opt_by_keyword(tokens, K_UPTIME))) {
+    tor_assert(tok->n_args >= 1);
+    router->uptime = tor_parse_long(tok->args[0],10,0,LONG_MAX,&ok,NULL);
+    if (!ok) {
+      log_warn(LD_DIR, "Invalid uptime %s", escaped(tok->args[0]));
+      goto err;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_HIBERNATING))) {
+    tor_assert(tok->n_args >= 1);
+    router->is_hibernating
+      = (tor_parse_long(tok->args[0],10,0,LONG_MAX,NULL,NULL) != 0);
+  }
+
+  tok = find_by_keyword(tokens, K_PUBLISHED);
+  tor_assert(tok->n_args == 1);
+  if (parse_iso_time(tok->args[0], &router->cache_info.published_on) < 0)
+    goto err;
+
+  tok = find_by_keyword(tokens, K_ONION_KEY);
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_DIR,
+             "Relay's onion key had invalid exponent.");
+    goto err;
+  }
+  router->onion_pkey = tok->key;
+  tok->key = NULL; /* Prevent free */
+
+  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+    curve25519_public_key_t k;
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus ntor-onion-key in routerinfo");
+      goto err;
+    }
+    router->onion_curve25519_pkey =
+      tor_memdup(&k, sizeof(curve25519_public_key_t));
+  }
+
+  tok = find_by_keyword(tokens, K_SIGNING_KEY);
+  router->identity_pkey = tok->key;
+  tok->key = NULL; /* Prevent free */
+  if (crypto_pk_get_digest(router->identity_pkey,
+                           router->cache_info.identity_digest)) {
+    log_warn(LD_DIR, "Couldn't calculate key digest"); goto err;
+  }
+
+  {
+    directory_token_t *ed_sig_tok, *ed_cert_tok, *cc_tap_tok, *cc_ntor_tok,
+      *master_key_tok;
+    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
+    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
+    master_key_tok = find_opt_by_keyword(tokens, K_MASTER_KEY_ED25519);
+    cc_tap_tok = find_opt_by_keyword(tokens, K_ONION_KEY_CROSSCERT);
+    cc_ntor_tok = find_opt_by_keyword(tokens, K_NTOR_ONION_KEY_CROSSCERT);
+    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok +
+      !!cc_tap_tok + !!cc_ntor_tok;
+    if ((n_ed_toks != 0 && n_ed_toks != 4) ||
+        (n_ed_toks == 4 && !router->onion_curve25519_pkey)) {
+      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
+               "cross-certification support");
+      goto err;
+    }
+    if (master_key_tok && !ed_sig_tok) {
+      log_warn(LD_DIR, "Router descriptor has ed25519 master key but no "
+               "certificate");
+      goto err;
+    }
+    if (ed_sig_tok) {
+      tor_assert(ed_cert_tok && cc_tap_tok && cc_ntor_tok);
+      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
+      if (ed_cert_token_pos == -1 || router_token_pos == -1 ||
+          (ed_cert_token_pos != router_token_pos + 1 &&
+           ed_cert_token_pos != router_token_pos - 1)) {
+        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
+        goto err;
+      }
+      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
+        log_warn(LD_DIR, "Ed25519 signature in wrong position");
+        goto err;
+      }
+      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_ntor_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on ntor-onion-key-crosscert "
+                 "in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_tap_tok->object_type, "CROSSCERT")) {
+        log_warn(LD_DIR, "Wrong object type on onion-key-crosscert "
+                 "in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_ntor_tok->args[0], "0") &&
+          strcmp(cc_ntor_tok->args[0], "1")) {
+        log_warn(LD_DIR, "Bad sign bit on ntor-onion-key-crosscert");
+        goto err;
+      }
+      int ntor_cc_sign_bit = !strcmp(cc_ntor_tok->args[0], "1");
+
+      uint8_t d256[DIGEST256_LEN];
+      const char *signed_start, *signed_end;
+      tor_cert_t *cert = tor_cert_parse(
+                       (const uint8_t*)ed_cert_tok->object_body,
+                       ed_cert_tok->object_size);
+      if (! cert) {
+        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
+        goto err;
+      }
+      /* makes sure it gets freed. */
+      router->cache_info.signing_key_cert = cert;
+
+      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
+          ! cert->signing_key_included) {
+        log_warn(LD_DIR, "Invalid form for ed25519 cert");
+        goto err;
+      }
+
+      if (master_key_tok) {
+        /* This token is optional, but if it's present, it must match
+         * the signature in the signing cert, or supplant it. */
+        tor_assert(master_key_tok->n_args >= 1);
+        ed25519_public_key_t pkey;
+        if (ed25519_public_from_base64(&pkey, master_key_tok->args[0])<0) {
+          log_warn(LD_DIR, "Can't parse ed25519 master key");
+          goto err;
+        }
+
+        if (fast_memneq(&cert->signing_key.pubkey,
+                        pkey.pubkey, ED25519_PUBKEY_LEN)) {
+          log_warn(LD_DIR, "Ed25519 master key does not match "
+                   "key in certificate");
+          goto err;
+        }
+      }
+      ntor_cc_cert = tor_cert_parse((const uint8_t*)cc_ntor_tok->object_body,
+                                    cc_ntor_tok->object_size);
+      if (!ntor_cc_cert) {
+        log_warn(LD_DIR, "Couldn't parse ntor-onion-key-crosscert cert");
+        goto err;
+      }
+      if (ntor_cc_cert->cert_type != CERT_TYPE_ONION_ID ||
+          ! ed25519_pubkey_eq(&ntor_cc_cert->signed_key, &cert->signing_key)) {
+        log_warn(LD_DIR, "Invalid contents for ntor-onion-key-crosscert cert");
+        goto err;
+      }
+
+      ed25519_public_key_t ntor_cc_pk;
+      if (ed25519_public_key_from_curve25519_public_key(&ntor_cc_pk,
+                                            router->onion_curve25519_pkey,
+                                            ntor_cc_sign_bit)<0) {
+        log_warn(LD_DIR, "Error converting onion key to ed25519");
+        goto err;
+      }
+
+      if (router_get_hash_impl_helper(s, end-s, "router ",
+                                      "\nrouter-sig-ed25519",
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
+        log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor");
+        goto err;
+      }
+      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
+        strlen(ED_DESC_SIGNATURE_PREFIX));
+      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
+      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
+      crypto_digest_free(d);
+
+      ed25519_checkable_t check[3];
+      int check_ok[3];
+      time_t expires = TIME_MAX;
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, &expires) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
+        goto err;
+      }
+      if (tor_cert_get_checkable_sig(&check[1],
+                               ntor_cc_cert, &ntor_cc_pk, &expires) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for ntor_cc_cert.");
+        goto err;
+      }
+
+      if (ed25519_signature_from_base64(&check[2].signature,
+                                        ed_sig_tok->args[0])<0) {
+        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
+        goto err;
+      }
+      check[2].pubkey = &cert->signed_key;
+      check[2].msg = d256;
+      check[2].len = DIGEST256_LEN;
+
+      if (ed25519_checksig_batch(check_ok, check, 3) < 0) {
+        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
+        goto err;
+      }
+
+      if (check_tap_onion_key_crosscert(
+                      (const uint8_t*)cc_tap_tok->object_body,
+                      (int)cc_tap_tok->object_size,
+                      router->onion_pkey,
+                      &cert->signing_key,
+                      (const uint8_t*)router->cache_info.identity_digest)<0) {
+        log_warn(LD_DIR, "Incorrect TAP cross-verification");
+        goto err;
+      }
+
+      /* We check this before adding it to the routerlist. */
+      router->cert_expiration_time = expires;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_FINGERPRINT))) {
+    /* If there's a fingerprint line, it must match the identity digest. */
+    char d[DIGEST_LEN];
+    tor_assert(tok->n_args == 1);
+    tor_strstrip(tok->args[0], " ");
+    if (base16_decode(d, DIGEST_LEN,
+                      tok->args[0], strlen(tok->args[0])) != DIGEST_LEN) {
+      log_warn(LD_DIR, "Couldn't decode router fingerprint %s",
+               escaped(tok->args[0]));
+      goto err;
+    }
+    if (tor_memneq(d,router->cache_info.identity_digest, DIGEST_LEN)) {
+      log_warn(LD_DIR, "Fingerprint '%s' does not match identity digest.",
+               tok->args[0]);
+      goto err;
+    }
+  }
+
+  {
+    const char *version = NULL, *protocols = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PLATFORM))) {
+      router->platform = tor_strdup(tok->args[0]);
+      version = tok->args[0];
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      router->protocol_list = tor_strdup(tok->args[0]);
+      protocols = tok->args[0];
+    }
+
+    summarize_protover_flags(&router->pv, protocols, version);
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) {
+    router->contact_info = tor_strdup(tok->args[0]);
+  }
+
+  if (find_opt_by_keyword(tokens, K_REJECT6) ||
+      find_opt_by_keyword(tokens, K_ACCEPT6)) {
+    log_warn(LD_DIR, "Rejecting router with reject6/accept6 line: they crash "
+             "older Tors.");
+    goto err;
+  }
+  {
+    smartlist_t *or_addresses = find_all_by_keyword(tokens, K_OR_ADDRESS);
+    if (or_addresses) {
+      find_single_ipv6_orport(or_addresses, &router->ipv6_addr,
+                              &router->ipv6_orport);
+      smartlist_free(or_addresses);
+    }
+  }
+  exit_policy_tokens = find_all_exitpolicy(tokens);
+  if (!smartlist_len(exit_policy_tokens)) {
+    log_warn(LD_DIR, "No exit policy tokens in descriptor.");
+    goto err;
+  }
+  SMARTLIST_FOREACH(exit_policy_tokens, directory_token_t *, t,
+                    if (router_add_exit_policy(router,t)<0) {
+                      log_warn(LD_DIR,"Error in exit policy");
+                      goto err;
+                    });
+  policy_expand_private(&router->exit_policy);
+
+  if ((tok = find_opt_by_keyword(tokens, K_IPV6_POLICY)) && tok->n_args) {
+    router->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+    if (! router->ipv6_exit_policy) {
+      log_warn(LD_DIR , "Error in ipv6-policy %s", escaped(tok->args[0]));
+      goto err;
+    }
+  }
+
+  if (policy_is_reject_star(router->exit_policy, AF_INET, 1) &&
+      (!router->ipv6_exit_policy ||
+       short_policy_is_reject_star(router->ipv6_exit_policy)))
+    router->policy_is_reject_star = 1;
+
+  if ((tok = find_opt_by_keyword(tokens, K_FAMILY)) && tok->n_args) {
+    int i;
+    router->declared_family = smartlist_new();
+    for (i=0;i<tok->n_args;++i) {
+      if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+        log_warn(LD_DIR, "Illegal nickname %s in family line",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+      smartlist_add_strdup(router->declared_family, tok->args[i]);
+    }
+  }
+
+  if (find_opt_by_keyword(tokens, K_CACHES_EXTRA_INFO))
+    router->caches_extra_info = 1;
+
+  if (find_opt_by_keyword(tokens, K_ALLOW_SINGLE_HOP_EXITS))
+    router->allow_single_hop_exits = 1;
+
+  if ((tok = find_opt_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) {
+    tor_assert(tok->n_args >= 1);
+    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
+      if (base16_decode(router->cache_info.extra_info_digest, DIGEST_LEN,
+                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) {
+          log_warn(LD_DIR,"Invalid extra info digest");
+      }
+    } else {
+      log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0]));
+    }
+
+    if (tok->n_args >= 2) {
+      if (digest256_from_base64(router->cache_info.extra_info_digest256,
+                                tok->args[1]) < 0) {
+        log_warn(LD_DIR, "Invalid extra info digest256 %s",
+                 escaped(tok->args[1]));
+      }
+    }
+  }
+
+  if (find_opt_by_keyword(tokens, K_HIDDEN_SERVICE_DIR)) {
+    router->wants_to_be_hs_dir = 1;
+  }
+
+  /* This router accepts tunnelled directory requests via begindir if it has
+   * an open dirport or it included "tunnelled-dir-server". */
+  if (find_opt_by_keyword(tokens, K_DIR_TUNNELLED) || router->dir_port > 0) {
+    router->supports_tunnelled_dir_requests = 1;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
+#ifdef COUNT_DISTINCT_DIGESTS
+  if (!verified_digests)
+    verified_digests = digestmap_new();
+  digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1);
+#endif
+
+  if (!router->or_port) {
+    log_warn(LD_DIR,"or_port unreadable or 0. Failing.");
+    goto err;
+  }
+
+  /* We've checked everything that's covered by the hash. */
+  can_dl_again = 1;
+  if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0,
+                            "router descriptor") < 0)
+    goto err;
+
+  if (!router->platform) {
+    router->platform = tor_strdup("<unknown>");
+  }
+  goto done;
+
+ err:
+  dump_desc(s_dup, "router descriptor");
+  routerinfo_free(router);
+  router = NULL;
+ done:
+  tor_cert_free(ntor_cc_cert);
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  smartlist_free(exit_policy_tokens);
+  if (area) {
+    DUMP_AREA(area, "routerinfo");
+    memarea_drop_all(area);
+  }
+  if (can_dl_again_out)
+    *can_dl_again_out = can_dl_again;
+  return router;
+}
+
+/** Parse a single extrainfo entry from the string <b>s</b>, ending at
+ * <b>end</b>.  (If <b>end</b> is NULL, parse up to the end of <b>s</b>.)  If
+ * <b>cache_copy</b> is true, make a copy of the extra-info document in the
+ * cache_info fields of the result.  If <b>routermap</b> is provided, use it
+ * as a map from router identity to routerinfo_t when looking up signing keys.
+ *
+ * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
+ * if it's okay to try to download an extrainfo with this same digest again,
+ * and 0 if it isn't.  (It might not be okay to download it again if part of
+ * the part covered by the digest is invalid.)
+ */
+extrainfo_t *
+extrainfo_parse_entry_from_string(const char *s, const char *end,
+                            int cache_copy, struct digest_ri_map_t *routermap,
+                            int *can_dl_again_out)
+{
+  extrainfo_t *extrainfo = NULL;
+  char digest[128];
+  smartlist_t *tokens = NULL;
+  directory_token_t *tok;
+  crypto_pk_t *key = NULL;
+  routerinfo_t *router = NULL;
+  memarea_t *area = NULL;
+  const char *s_dup = s;
+  /* Do not set this to '1' until we have parsed everything that we intend to
+   * parse that's covered by the hash. */
+  int can_dl_again = 0;
+
+  if (BUG(s == NULL))
+    return NULL;
+
+  if (!end) {
+    end = s + strlen(s);
+  }
+
+  /* point 'end' to a point immediately after the final newline. */
+  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
+    --end;
+
+  if (router_get_extrainfo_hash(s, end-s, digest) < 0) {
+    log_warn(LD_DIR, "Couldn't compute router hash.");
+    goto err;
+  }
+  tokens = smartlist_new();
+  area = memarea_new();
+  if (tokenize_string(area,s,end,tokens,extrainfo_token_table,0)) {
+    log_warn(LD_DIR, "Error tokenizing extra-info document.");
+    goto err;
+  }
+
+  if (smartlist_len(tokens) < 2) {
+    log_warn(LD_DIR, "Impossibly short extra-info document.");
+    goto err;
+  }
+
+  /* XXXX Accept this in position 1 too, and ed identity in position 0. */
+  tok = smartlist_get(tokens,0);
+  if (tok->tp != K_EXTRA_INFO) {
+    log_warn(LD_DIR,"Entry does not start with \"extra-info\"");
+    goto err;
+  }
+
+  extrainfo = tor_malloc_zero(sizeof(extrainfo_t));
+  extrainfo->cache_info.is_extrainfo = 1;
+  if (cache_copy)
+    extrainfo->cache_info.signed_descriptor_body = tor_memdup_nulterm(s,end-s);
+  extrainfo->cache_info.signed_descriptor_len = end-s;
+  memcpy(extrainfo->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+  crypto_digest256((char*)extrainfo->digest256, s, end-s, DIGEST_SHA256);
+
+  tor_assert(tok->n_args >= 2);
+  if (!is_legal_nickname(tok->args[0])) {
+    log_warn(LD_DIR,"Bad nickname %s on \"extra-info\"",escaped(tok->args[0]));
+    goto err;
+  }
+  strlcpy(extrainfo->nickname, tok->args[0], sizeof(extrainfo->nickname));
+  if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
+      base16_decode(extrainfo->cache_info.identity_digest, DIGEST_LEN,
+                    tok->args[1], HEX_DIGEST_LEN) != DIGEST_LEN) {
+    log_warn(LD_DIR,"Invalid fingerprint %s on \"extra-info\"",
+             escaped(tok->args[1]));
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_PUBLISHED);
+  if (parse_iso_time(tok->args[0], &extrainfo->cache_info.published_on)) {
+    log_warn(LD_DIR,"Invalid published time %s on \"extra-info\"",
+             escaped(tok->args[0]));
+    goto err;
+  }
+
+  {
+    directory_token_t *ed_sig_tok, *ed_cert_tok;
+    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
+    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
+    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok;
+    if (n_ed_toks != 0 && n_ed_toks != 2) {
+      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
+               "cross-certification support");
+      goto err;
+    }
+    if (ed_sig_tok) {
+      tor_assert(ed_cert_tok);
+      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
+      if (ed_cert_token_pos != 1) {
+        /* Accept this in position 0 XXXX */
+        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
+        goto err;
+      }
+      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
+        log_warn(LD_DIR, "Ed25519 signature in wrong position");
+        goto err;
+      }
+      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
+        goto err;
+      }
+
+      uint8_t d256[DIGEST256_LEN];
+      const char *signed_start, *signed_end;
+      tor_cert_t *cert = tor_cert_parse(
+                       (const uint8_t*)ed_cert_tok->object_body,
+                       ed_cert_tok->object_size);
+      if (! cert) {
+        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
+        goto err;
+      }
+      /* makes sure it gets freed. */
+      extrainfo->cache_info.signing_key_cert = cert;
+
+      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
+          ! cert->signing_key_included) {
+        log_warn(LD_DIR, "Invalid form for ed25519 cert");
+        goto err;
+      }
+
+      if (router_get_hash_impl_helper(s, end-s, "extra-info ",
+                                      "\nrouter-sig-ed25519",
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
+        log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo");
+        goto err;
+      }
+      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
+        strlen(ED_DESC_SIGNATURE_PREFIX));
+      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
+      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
+      crypto_digest_free(d);
+
+      ed25519_checkable_t check[2];
+      int check_ok[2];
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, NULL) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
+        goto err;
+      }
+
+      if (ed25519_signature_from_base64(&check[1].signature,
+                                        ed_sig_tok->args[0])<0) {
+        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
+        goto err;
+      }
+      check[1].pubkey = &cert->signed_key;
+      check[1].msg = d256;
+      check[1].len = DIGEST256_LEN;
+
+      if (ed25519_checksig_batch(check_ok, check, 2) < 0) {
+        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
+        goto err;
+      }
+      /* We don't check the certificate expiration time: checking that it
+       * matches the cert in the router descriptor is adequate. */
+    }
+  }
+
+  /* We've checked everything that's covered by the hash. */
+  can_dl_again = 1;
+
+  if (routermap &&
+      (router = digestmap_get((digestmap_t*)routermap,
+                              extrainfo->cache_info.identity_digest))) {
+    key = router->identity_pkey;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
+  if (strcmp(tok->object_type, "SIGNATURE") ||
+      tok->object_size < 128 || tok->object_size > 512) {
+    log_warn(LD_DIR, "Bad object type or length on extra-info signature");
+    goto err;
+  }
+
+  if (key) {
+    if (check_signature_token(digest, DIGEST_LEN, tok, key, 0,
+                              "extra-info") < 0)
+      goto err;
+
+    if (router)
+      extrainfo->cache_info.send_unencrypted =
+        router->cache_info.send_unencrypted;
+  } else {
+    extrainfo->pending_sig = tor_memdup(tok->object_body,
+                                        tok->object_size);
+    extrainfo->pending_sig_len = tok->object_size;
+  }
+
+  goto done;
+ err:
+  dump_desc(s_dup, "extra-info descriptor");
+  extrainfo_free(extrainfo);
+  extrainfo = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    DUMP_AREA(area, "extrainfo");
+    memarea_drop_all(area);
+  }
+  if (can_dl_again_out)
+    *can_dl_again_out = can_dl_again;
+  return extrainfo;
+}
+
+/** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
+ * the first character after the certificate. */
+authority_cert_t *
+authority_cert_parse_from_string(const char *s, const char **end_of_string)
+{
+  /** Reject any certificate at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_CERT_SIZE (128*1024)
+
+  authority_cert_t *cert = NULL, *old_cert;
+  smartlist_t *tokens = NULL;
+  char digest[DIGEST_LEN];
+  directory_token_t *tok;
+  char fp_declared[DIGEST_LEN];
+  char *eos;
+  size_t len;
+  int found;
+  memarea_t *area = NULL;
+  const char *s_dup = s;
+
+  s = eat_whitespace(s);
+  eos = strstr(s, "\ndir-key-certification");
+  if (! eos) {
+    log_warn(LD_DIR, "No signature found on key certificate");
+    return NULL;
+  }
+  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
+  if (! eos) {
+    log_warn(LD_DIR, "No end-of-signature found on key certificate");
+    return NULL;
+  }
+  eos = strchr(eos+2, '\n');
+  tor_assert(eos);
+  ++eos;
+  len = eos - s;
+
+  if (len > MAX_CERT_SIZE) {
+    log_warn(LD_DIR, "Certificate is far too big (at %lu bytes long); "
+             "rejecting", (unsigned long)len);
+    return NULL;
+  }
+
+  tokens = smartlist_new();
+  area = memarea_new();
+  if (tokenize_string(area,s, eos, tokens, dir_key_certificate_table, 0) < 0) {
+    log_warn(LD_DIR, "Error tokenizing key certificate");
+    goto err;
+  }
+  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
+                           "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
+    goto err;
+  tok = smartlist_get(tokens, 0);
+  if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) {
+    log_warn(LD_DIR,
+             "Key certificate does not begin with a recognized version (3).");
+    goto err;
+  }
+
+  cert = tor_malloc_zero(sizeof(authority_cert_t));
+  memcpy(cert->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+
+  tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY);
+  tor_assert(tok->key);
+  cert->signing_key = tok->key;
+  tok->key = NULL;
+  if (crypto_pk_get_digest(cert->signing_key, cert->signing_key_digest))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_DIR_IDENTITY_KEY);
+  tor_assert(tok->key);
+  cert->identity_key = tok->key;
+  tok->key = NULL;
+
+  tok = find_by_keyword(tokens, K_FINGERPRINT);
+  tor_assert(tok->n_args);
+  if (base16_decode(fp_declared, DIGEST_LEN, tok->args[0],
+                    strlen(tok->args[0])) != DIGEST_LEN) {
+    log_warn(LD_DIR, "Couldn't decode key certificate fingerprint %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+
+  if (crypto_pk_get_digest(cert->identity_key,
+                           cert->cache_info.identity_digest))
+    goto err;
+
+  if (tor_memneq(cert->cache_info.identity_digest, fp_declared, DIGEST_LEN)) {
+    log_warn(LD_DIR, "Digest of certificate key didn't match declared "
+             "fingerprint");
+    goto err;
+  }
+
+  tok = find_opt_by_keyword(tokens, K_DIR_ADDRESS);
+  if (tok) {
+    struct in_addr in;
+    char *address = NULL;
+    tor_assert(tok->n_args);
+    /* XXX++ use some tor_addr parse function below instead. -RD */
+    if (tor_addr_port_split(LOG_WARN, tok->args[0], &address,
+                            &cert->dir_port) < 0 ||
+        tor_inet_aton(address, &in) == 0) {
+      log_warn(LD_DIR, "Couldn't parse dir-address in certificate");
+      tor_free(address);
+      goto err;
+    }
+    cert->addr = ntohl(in.s_addr);
+    tor_free(address);
+  }
+
+  tok = find_by_keyword(tokens, K_DIR_KEY_PUBLISHED);
+  if (parse_iso_time(tok->args[0], &cert->cache_info.published_on) < 0) {
+     goto err;
+  }
+  tok = find_by_keyword(tokens, K_DIR_KEY_EXPIRES);
+  if (parse_iso_time(tok->args[0], &cert->expires) < 0) {
+     goto err;
+  }
+
+  tok = smartlist_get(tokens, smartlist_len(tokens)-1);
+  if (tok->tp != K_DIR_KEY_CERTIFICATION) {
+    log_warn(LD_DIR, "Certificate didn't end with dir-key-certification.");
+    goto err;
+  }
+
+  /* If we already have this cert, don't bother checking the signature. */
+  old_cert = authority_cert_get_by_digests(
+                                     cert->cache_info.identity_digest,
+                                     cert->signing_key_digest);
+  found = 0;
+  if (old_cert) {
+    /* XXXX We could just compare signed_descriptor_digest, but that wouldn't
+     * buy us much. */
+    if (old_cert->cache_info.signed_descriptor_len == len &&
+        old_cert->cache_info.signed_descriptor_body &&
+        tor_memeq(s, old_cert->cache_info.signed_descriptor_body, len)) {
+      log_debug(LD_DIR, "We already checked the signature on this "
+                "certificate; no need to do so again.");
+      found = 1;
+    }
+  }
+  if (!found) {
+    if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0,
+                              "key certificate")) {
+      goto err;
+    }
+
+    tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT);
+    if (check_signature_token(cert->cache_info.identity_digest,
+                              DIGEST_LEN,
+                              tok,
+                              cert->signing_key,
+                              CST_NO_CHECK_OBJTYPE,
+                              "key cross-certification")) {
+      goto err;
+    }
+  }
+
+  cert->cache_info.signed_descriptor_len = len;
+  cert->cache_info.signed_descriptor_body = tor_malloc(len+1);
+  memcpy(cert->cache_info.signed_descriptor_body, s, len);
+  cert->cache_info.signed_descriptor_body[len] = 0;
+  cert->cache_info.saved_location = SAVED_NOWHERE;
+
+  if (end_of_string) {
+    *end_of_string = eat_whitespace(eos);
+  }
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "authority cert");
+    memarea_drop_all(area);
+  }
+  return cert;
+ err:
+  dump_desc(s_dup, "authority cert");
+  authority_cert_free(cert);
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "authority cert");
+    memarea_drop_all(area);
+  }
+  return NULL;
+}
+
+/** Helper: given a string <b>s</b>, return the start of the next router-status
+ * object (starting with "r " at the start of a line).  If none is found,
+ * return the start of the directory footer, or the next directory signature.
+ * If none is found, return the end of the string. */
+static inline const char *
+find_start_of_next_routerstatus(const char *s)
+{
+  const char *eos, *footer, *sig;
+  if ((eos = strstr(s, "\nr ")))
+    ++eos;
+  else
+    eos = s + strlen(s);
+
+  footer = tor_memstr(s, eos-s, "\ndirectory-footer");
+  sig = tor_memstr(s, eos-s, "\ndirectory-signature");
+
+  if (footer && sig)
+    return MIN(footer, sig) + 1;
+  else if (footer)
+    return footer+1;
+  else if (sig)
+    return sig+1;
+  else
+    return eos;
+}
+
+/** Parse the GuardFraction string from a consensus or vote.
+ *
+ *  If <b>vote</b> or <b>vote_rs</b> are set the document getting
+ *  parsed is a vote routerstatus. Otherwise it's a consensus. This is
+ *  the same semantic as in routerstatus_parse_entry_from_string(). */
+STATIC int
+routerstatus_parse_guardfraction(const char *guardfraction_str,
+                                 networkstatus_t *vote,
+                                 vote_routerstatus_t *vote_rs,
+                                 routerstatus_t *rs)
+{
+  int ok;
+  const char *end_of_header = NULL;
+  int is_consensus = !vote_rs;
+  uint32_t guardfraction;
+
+  tor_assert(bool_eq(vote, vote_rs));
+
+  /* If this info comes from a consensus, but we should't apply
+     guardfraction, just exit. */
+  if (is_consensus && !should_apply_guardfraction(NULL)) {
+    return 0;
+  }
+
+  end_of_header = strchr(guardfraction_str, '=');
+  if (!end_of_header) {
+    return -1;
+  }
+
+  guardfraction = (uint32_t)tor_parse_ulong(end_of_header+1,
+                                            10, 0, 100, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid GuardFraction %s", escaped(guardfraction_str));
+    return -1;
+  }
+
+  log_debug(LD_GENERAL, "[*] Parsed %s guardfraction '%s' for '%s'.",
+            is_consensus ? "consensus" : "vote",
+            guardfraction_str, rs->nickname);
+
+  if (!is_consensus) { /* We are parsing a vote */
+    vote_rs->status.guardfraction_percentage = guardfraction;
+    vote_rs->status.has_guardfraction = 1;
+  } else {
+    /* We are parsing a consensus. Only apply guardfraction to guards. */
+    if (rs->is_possible_guard) {
+      rs->guardfraction_percentage = guardfraction;
+      rs->has_guardfraction = 1;
+    } else {
+      log_warn(LD_BUG, "Got GuardFraction for non-guard %s. "
+               "This is not supposed to happen. Not applying. ", rs->nickname);
+    }
+  }
+
+  return 0;
+}
+
+/** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
+ * falling back or correcting them based on <b>version</b> as appropriate.
+ */
+STATIC void
+summarize_protover_flags(protover_summary_flags_t *out,
+                         const char *protocols,
+                         const char *version)
+{
+  tor_assert(out);
+  memset(out, 0, sizeof(*out));
+  if (protocols) {
+    out->protocols_known = 1;
+    out->supports_extend2_cells =
+      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+    out->supports_ed25519_link_handshake_compat =
+      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_link_handshake_any =
+      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_hs_intro =
+      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+    out->supports_v3_hsdir =
+      protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                      PROTOVER_HSDIR_V3);
+    out->supports_v3_rendezvous_point =
+      protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  }
+  if (version && !strcmpstart(version, "Tor ")) {
+    if (!out->protocols_known) {
+      /* The version is a "Tor" version, and where there is no
+       * list of protocol versions that we should be looking at instead. */
+
+      out->supports_extend2_cells =
+        tor_version_as_new_as(version, "0.2.4.8-alpha");
+      out->protocols_known = 1;
+    } else {
+      /* Bug #22447 forces us to filter on this version. */
+      if (!tor_version_as_new_as(version, "0.3.0.8")) {
+        out->supports_v3_hsdir = 0;
+      }
+    }
+  }
+}
+
+/** Given a string at *<b>s</b>, containing a routerstatus object, and an
+ * empty smartlist at <b>tokens</b>, parse and return the first router status
+ * object in the string, and advance *<b>s</b> to just after the end of the
+ * router status.  Return NULL and advance *<b>s</b> on error.
+ *
+ * If <b>vote</b> and <b>vote_rs</b> are provided, don't allocate a fresh
+ * routerstatus but use <b>vote_rs</b> instead.
+ *
+ * If <b>consensus_method</b> is nonzero, this routerstatus is part of a
+ * consensus, and we should parse it according to the method used to
+ * make that consensus.
+ *
+ * Parse according to the syntax used by the consensus flavor <b>flav</b>.
+ **/
+STATIC routerstatus_t *
+routerstatus_parse_entry_from_string(memarea_t *area,
+                                     const char **s, smartlist_t *tokens,
+                                     networkstatus_t *vote,
+                                     vote_routerstatus_t *vote_rs,
+                                     int consensus_method,
+                                     consensus_flavor_t flav)
+{
+  const char *eos, *s_dup = *s;
+  routerstatus_t *rs = NULL;
+  directory_token_t *tok;
+  char timebuf[ISO_TIME_LEN+1];
+  struct in_addr in;
+  int offset = 0;
+  tor_assert(tokens);
+  tor_assert(bool_eq(vote, vote_rs));
+
+  if (!consensus_method)
+    flav = FLAV_NS;
+  tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
+
+  eos = find_start_of_next_routerstatus(*s);
+
+  if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
+    log_warn(LD_DIR, "Error tokenizing router status");
+    goto err;
+  }
+  if (smartlist_len(tokens) < 1) {
+    log_warn(LD_DIR, "Impossibly short router status");
+    goto err;
+  }
+  tok = find_by_keyword(tokens, K_R);
+  tor_assert(tok->n_args >= 7); /* guaranteed by GE(7) in K_R setup */
+  if (flav == FLAV_NS) {
+    if (tok->n_args < 8) {
+      log_warn(LD_DIR, "Too few arguments to r");
+      goto err;
+    }
+  } else if (flav == FLAV_MICRODESC) {
+    offset = -1; /* There is no descriptor digest in an md consensus r line */
+  }
+
+  if (vote_rs) {
+    rs = &vote_rs->status;
+  } else {
+    rs = tor_malloc_zero(sizeof(routerstatus_t));
+  }
+
+  if (!is_legal_nickname(tok->args[0])) {
+    log_warn(LD_DIR,
+             "Invalid nickname %s in router status; skipping.",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  strlcpy(rs->nickname, tok->args[0], sizeof(rs->nickname));
+
+  if (digest_from_base64(rs->identity_digest, tok->args[1])) {
+    log_warn(LD_DIR, "Error decoding identity digest %s",
+             escaped(tok->args[1]));
+    goto err;
+  }
+
+  if (flav == FLAV_NS) {
+    if (digest_from_base64(rs->descriptor_digest, tok->args[2])) {
+      log_warn(LD_DIR, "Error decoding descriptor digest %s",
+               escaped(tok->args[2]));
+      goto err;
+    }
+  }
+
+  if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s",
+                   tok->args[3+offset], tok->args[4+offset]) < 0 ||
+      parse_iso_time(timebuf, &rs->published_on)<0) {
+    log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]",
+             tok->args[3+offset], tok->args[4+offset],
+             offset, (int)flav);
+    goto err;
+  }
+
+  if (tor_inet_aton(tok->args[5+offset], &in) == 0) {
+    log_warn(LD_DIR, "Error parsing router address in network-status %s",
+             escaped(tok->args[5+offset]));
+    goto err;
+  }
+  rs->addr = ntohl(in.s_addr);
+
+  rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset],
+                                         10,0,65535,NULL,NULL);
+  rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset],
+                                           10,0,65535,NULL,NULL);
+
+  {
+    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+    if (a_lines) {
+      find_single_ipv6_orport(a_lines, &rs->ipv6_addr, &rs->ipv6_orport);
+      smartlist_free(a_lines);
+    }
+  }
+
+  tok = find_opt_by_keyword(tokens, K_S);
+  if (tok && vote) {
+    int i;
+    vote_rs->flags = 0;
+    for (i=0; i < tok->n_args; ++i) {
+      int p = smartlist_string_pos(vote->known_flags, tok->args[i]);
+      if (p >= 0) {
+        vote_rs->flags |= (UINT64_C(1)<<p);
+      } else {
+        log_warn(LD_DIR, "Flags line had a flag %s not listed in known_flags.",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+    }
+  } else if (tok) {
+    /* This is a consensus, not a vote. */
+    int i;
+    for (i=0; i < tok->n_args; ++i) {
+      if (!strcmp(tok->args[i], "Exit"))
+        rs->is_exit = 1;
+      else if (!strcmp(tok->args[i], "Stable"))
+        rs->is_stable = 1;
+      else if (!strcmp(tok->args[i], "Fast"))
+        rs->is_fast = 1;
+      else if (!strcmp(tok->args[i], "Running"))
+        rs->is_flagged_running = 1;
+      else if (!strcmp(tok->args[i], "Named"))
+        rs->is_named = 1;
+      else if (!strcmp(tok->args[i], "Valid"))
+        rs->is_valid = 1;
+      else if (!strcmp(tok->args[i], "Guard"))
+        rs->is_possible_guard = 1;
+      else if (!strcmp(tok->args[i], "BadExit"))
+        rs->is_bad_exit = 1;
+      else if (!strcmp(tok->args[i], "Authority"))
+        rs->is_authority = 1;
+      else if (!strcmp(tok->args[i], "Unnamed") &&
+               consensus_method >= 2) {
+        /* Unnamed is computed right by consensus method 2 and later. */
+        rs->is_unnamed = 1;
+      } else if (!strcmp(tok->args[i], "HSDir")) {
+        rs->is_hs_dir = 1;
+      } else if (!strcmp(tok->args[i], "V2Dir")) {
+        rs->is_v2_dir = 1;
+      }
+    }
+    /* These are implied true by having been included in a consensus made
+     * with a given method */
+    rs->is_flagged_running = 1; /* Starting with consensus method 4. */
+    rs->is_valid = 1; /* Starting with consensus method 24. */
+  }
+  {
+    const char *protocols = NULL, *version = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      tor_assert(tok->n_args == 1);
+      protocols = tok->args[0];
+    }
+    if ((tok = find_opt_by_keyword(tokens, K_V))) {
+      tor_assert(tok->n_args == 1);
+      version = tok->args[0];
+      if (vote_rs) {
+        vote_rs->version = tor_strdup(tok->args[0]);
+      }
+    }
+
+    summarize_protover_flags(&rs->pv, protocols, version);
+  }
+
+  /* handle weighting/bandwidth info */
+  if ((tok = find_opt_by_keyword(tokens, K_W))) {
+    int i;
+    for (i=0; i < tok->n_args; ++i) {
+      if (!strcmpstart(tok->args[i], "Bandwidth=")) {
+        int ok;
+        rs->bandwidth_kb =
+          (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
+                                    10, 0, UINT32_MAX,
+                                    &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Invalid Bandwidth %s", escaped(tok->args[i]));
+          goto err;
+        }
+        rs->has_bandwidth = 1;
+      } else if (!strcmpstart(tok->args[i], "Measured=") && vote_rs) {
+        int ok;
+        vote_rs->measured_bw_kb =
+            (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
+                                      10, 0, UINT32_MAX, &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Invalid Measured Bandwidth %s",
+                   escaped(tok->args[i]));
+          goto err;
+        }
+        vote_rs->has_measured_bw = 1;
+        vote->has_measured_bws = 1;
+      } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) {
+        rs->bw_is_unmeasured = 1;
+      } else if (!strcmpstart(tok->args[i], "GuardFraction=")) {
+        if (routerstatus_parse_guardfraction(tok->args[i],
+                                             vote, vote_rs, rs) < 0) {
+          goto err;
+        }
+      }
+    }
+  }
+
+  /* parse exit policy summaries */
+  if ((tok = find_opt_by_keyword(tokens, K_P))) {
+    tor_assert(tok->n_args == 1);
+    if (strcmpstart(tok->args[0], "accept ") &&
+        strcmpstart(tok->args[0], "reject ")) {
+      log_warn(LD_DIR, "Unknown exit policy summary type %s.",
+               escaped(tok->args[0]));
+      goto err;
+    }
+    /* XXX weasel: parse this into ports and represent them somehow smart,
+     * maybe not here but somewhere on if we need it for the client.
+     * we should still parse it here to check it's valid tho.
+     */
+    rs->exitsummary = tor_strdup(tok->args[0]);
+    rs->has_exitsummary = 1;
+  }
+
+  if (vote_rs) {
+    SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) {
+      if (t->tp == K_M && t->n_args) {
+        vote_microdesc_hash_t *line =
+          tor_malloc(sizeof(vote_microdesc_hash_t));
+        line->next = vote_rs->microdesc;
+        line->microdesc_hash_line = tor_strdup(t->args[0]);
+        vote_rs->microdesc = line;
+      }
+      if (t->tp == K_ID) {
+        tor_assert(t->n_args >= 2);
+        if (!strcmp(t->args[0], "ed25519")) {
+          vote_rs->has_ed25519_listing = 1;
+          if (strcmp(t->args[1], "none") &&
+              digest256_from_base64((char*)vote_rs->ed25519_id,
+                                    t->args[1])<0) {
+            log_warn(LD_DIR, "Bogus ed25519 key in networkstatus vote");
+            goto err;
+          }
+        }
+      }
+      if (t->tp == K_PROTO) {
+        tor_assert(t->n_args == 1);
+        vote_rs->protocols = tor_strdup(t->args[0]);
+      }
+    } SMARTLIST_FOREACH_END(t);
+  } else if (flav == FLAV_MICRODESC) {
+    tok = find_opt_by_keyword(tokens, K_M);
+    if (tok) {
+      tor_assert(tok->n_args);
+      if (digest256_from_base64(rs->descriptor_digest, tok->args[0])) {
+        log_warn(LD_DIR, "Error decoding microdescriptor digest %s",
+                 escaped(tok->args[0]));
+        goto err;
+      }
+    } else {
+      log_info(LD_BUG, "Found an entry in networkstatus with no "
+               "microdescriptor digest. (Router %s ($%s) at %s:%d.)",
+               rs->nickname, hex_str(rs->identity_digest, DIGEST_LEN),
+               fmt_addr32(rs->addr), rs->or_port);
+    }
+  }
+
+  if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME))
+    rs->is_named = 0;
+
+  goto done;
+ err:
+  dump_desc(s_dup, "routerstatus entry");
+  if (rs && !vote_rs)
+    routerstatus_free(rs);
+  rs = NULL;
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_clear(tokens);
+  if (area) {
+    DUMP_AREA(area, "routerstatus entry");
+    memarea_clear(area);
+  }
+  *s = eos;
+
+  return rs;
+}
+
+int
+compare_vote_routerstatus_entries(const void **_a, const void **_b)
+{
+  const vote_routerstatus_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->status.identity_digest, b->status.identity_digest,
+                     DIGEST_LEN);
+}
+
+/** Verify the bandwidth weights of a network status document */
+int
+networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
+{
+  int64_t G=0, M=0, E=0, D=0, T=0;
+  double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed;
+  double Gtotal=0, Mtotal=0, Etotal=0;
+  const char *casename = NULL;
+  int valid = 1;
+  (void) consensus_method;
+
+  const int64_t weight_scale = networkstatus_get_weight_scale_param(ns);
+  tor_assert(weight_scale >= 1);
+  Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1);
+  Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1);
+  Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1);
+  Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1);
+  Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1);
+  Wme = networkstatus_get_bw_weight(ns, "Wme", -1);
+  Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1);
+  Weg = networkstatus_get_bw_weight(ns, "Weg", -1);
+  Wem = networkstatus_get_bw_weight(ns, "Wem", -1);
+  Wee = networkstatus_get_bw_weight(ns, "Wee", -1);
+  Wed = networkstatus_get_bw_weight(ns, "Wed", -1);
+
+  if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0
+          || Wem<0 || Wee<0 || Wed<0) {
+    log_warn(LD_BUG, "No bandwidth weights produced in consensus!");
+    return 0;
+  }
+
+  // First, sanity check basic summing properties that hold for all cases
+  // We use > 1 as the check for these because they are computed as integers.
+  // Sometimes there are rounding errors.
+  if (fabs(Wmm - weight_scale) > 1) {
+    log_warn(LD_BUG, "Wmm=%f != %"PRId64,
+             Wmm, (weight_scale));
+    valid = 0;
+  }
+
+  if (fabs(Wem - Wee) > 1) {
+    log_warn(LD_BUG, "Wem=%f != Wee=%f", Wem, Wee);
+    valid = 0;
+  }
+
+  if (fabs(Wgm - Wgg) > 1) {
+    log_warn(LD_BUG, "Wgm=%f != Wgg=%f", Wgm, Wgg);
+    valid = 0;
+  }
+
+  if (fabs(Weg - Wed) > 1) {
+    log_warn(LD_BUG, "Wed=%f != Weg=%f", Wed, Weg);
+    valid = 0;
+  }
+
+  if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wgg=%f != %"PRId64" - Wmg=%f", Wgg,
+             (weight_scale), Wmg);
+    valid = 0;
+  }
+
+  if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wee=%f != %"PRId64" - Wme=%f", Wee,
+             (weight_scale), Wme);
+    valid = 0;
+  }
+
+  if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wgd=%f + Wmd=%f + Wed=%f != %"PRId64,
+             Wgd, Wmd, Wed, (weight_scale));
+    valid = 0;
+  }
+
+  Wgg /= weight_scale;
+  Wgm /= weight_scale; (void) Wgm; // unused from here on.
+  Wgd /= weight_scale;
+
+  Wmg /= weight_scale;
+  Wmm /= weight_scale;
+  Wme /= weight_scale;
+  Wmd /= weight_scale;
+
+  Weg /= weight_scale; (void) Weg; // unused from here on.
+  Wem /= weight_scale; (void) Wem; // unused from here on.
+  Wee /= weight_scale;
+  Wed /= weight_scale;
+
+  // Then, gather G, M, E, D, T to determine case
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    int is_exit = 0;
+    /* Bug #2203: Don't count bad exits as exits for balancing */
+    is_exit = rs->is_exit && !rs->is_bad_exit;
+    if (rs->has_bandwidth) {
+      T += rs->bandwidth_kb;
+      if (is_exit && rs->is_possible_guard) {
+        D += rs->bandwidth_kb;
+        Gtotal += Wgd*rs->bandwidth_kb;
+        Mtotal += Wmd*rs->bandwidth_kb;
+        Etotal += Wed*rs->bandwidth_kb;
+      } else if (is_exit) {
+        E += rs->bandwidth_kb;
+        Mtotal += Wme*rs->bandwidth_kb;
+        Etotal += Wee*rs->bandwidth_kb;
+      } else if (rs->is_possible_guard) {
+        G += rs->bandwidth_kb;
+        Gtotal += Wgg*rs->bandwidth_kb;
+        Mtotal += Wmg*rs->bandwidth_kb;
+      } else {
+        M += rs->bandwidth_kb;
+        Mtotal += Wmm*rs->bandwidth_kb;
+      }
+    } else {
+      log_warn(LD_BUG, "Missing consensus bandwidth for router %s",
+               routerstatus_describe(rs));
+    }
+  } SMARTLIST_FOREACH_END(rs);
+
+  // Finally, check equality conditions depending upon case 1, 2 or 3
+  // Full equality cases: 1, 3b
+  // Partial equality cases: 2b (E=G), 3a (M=E)
+  // Fully unknown: 2a
+  if (3*E >= T && 3*G >= T) {
+    // Case 1: Neither are scarce
+    casename = "Case 1";
+    if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Etotal, Mtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+    if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Etotal, Gtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+    if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Mtotal, Gtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+  } else if (3*E < T && 3*G < T) {
+    int64_t R = MIN(E, G);
+    int64_t S = MAX(E, G);
+    /*
+     * Case 2: Both Guards and Exits are scarce
+     * Balance D between E and G, depending upon
+     * D capacity and scarcity. Devote no extra
+     * bandwidth to middle nodes.
+     */
+    if (R+D < S) { // Subcase a
+      double Rtotal, Stotal;
+      if (E < G) {
+        Rtotal = Etotal;
+        Stotal = Gtotal;
+      } else {
+        Rtotal = Gtotal;
+        Stotal = Etotal;
+      }
+      casename = "Case 2a";
+      // Rtotal < Stotal
+      if (Rtotal > Stotal) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Rtotal %f > Stotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Rtotal, Stotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Rtotal < T/3
+      if (3*Rtotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Rtotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Rtotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Stotal < T/3
+      if (3*Stotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Stotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Stotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Mtotal > T/3
+      if (3*Mtotal < T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Mtotal %f < T "
+                   "%"PRId64". "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Mtotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+    } else { // Subcase b: R+D > S
+      casename = "Case 2b";
+
+      /* Check the rare-M redirect case. */
+      if (D != 0 && 3*M < T) {
+        casename = "Case 2b (balanced)";
+        if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Mtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+        if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Mtotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      } else {
+        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      }
+    }
+  } else { // if (E < T/3 || G < T/3) {
+    int64_t S = MIN(E, G);
+    int64_t NS = MAX(E, G);
+    if (3*(S+D) < T) { // Subcase a:
+      double Stotal;
+      double NStotal;
+      if (G < E) {
+        casename = "Case 3a (G scarce)";
+        Stotal = Gtotal;
+        NStotal = Etotal;
+      } else { // if (G >= E) {
+        casename = "Case 3a (E scarce)";
+        NStotal = Gtotal;
+        Stotal = Etotal;
+      }
+      // Stotal < T/3
+      if (3*Stotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Stotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Stotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (NS >= M) {
+        if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: NStotal %f != Mtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, NStotal, Mtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      } else {
+        // if NS < M, NStotal > T/3 because only one of G or E is scarce
+        if (3*NStotal < T) {
+          log_warn(LD_DIR,
+                     "Bw Weight Failure for %s: 3*NStotal %f < T "
+                     "%"PRId64". G=%"PRId64" M=%"PRId64
+                     " E=%"PRId64" D=%"PRId64" T=%"PRId64". "
+                     "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                     casename, NStotal*3, (T),
+                     (G), (M), (E),
+                     (D), (T),
+                     Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      }
+    } else { // Subcase b: S+D >= T/3
+      casename = "Case 3b";
+      if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Etotal, Mtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Etotal, Gtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Mtotal, Gtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+    }
+  }
+
+  if (valid)
+    log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.",
+               casename);
+
+  return valid;
+}
+
+/** Check if a shared random value of type <b>srv_type</b> is in
+ *  <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return
+ *  -1 on failure, 0 on success. The resulting srv is allocated on the heap and
+ *  it's the responsibility of the caller to free it. */
+static int
+extract_one_srv(smartlist_t *tokens, directory_keyword srv_type,
+                sr_srv_t **srv_out)
+{
+  int ret = -1;
+  directory_token_t *tok;
+  sr_srv_t *srv = NULL;
+  smartlist_t *chunks;
+
+  tor_assert(tokens);
+
+  chunks = smartlist_new();
+  tok = find_opt_by_keyword(tokens, srv_type);
+  if (!tok) {
+    /* That's fine, no SRV is allowed. */
+    ret = 0;
+    goto end;
+  }
+  for (int i = 0; i < tok->n_args; i++) {
+    smartlist_add(chunks, tok->args[i]);
+  }
+  srv = sr_parse_srv(chunks);
+  if (srv == NULL) {
+    log_warn(LD_DIR, "SR: Unparseable SRV %s", escaped(tok->object_body));
+    goto end;
+  }
+  /* All is good. */
+  *srv_out = srv;
+  ret = 0;
+ end:
+  smartlist_free(chunks);
+  return ret;
+}
+
+/** Extract any shared random values found in <b>tokens</b> and place them in
+ *  the networkstatus <b>ns</b>. */
+static void
+extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
+{
+  const char *voter_identity;
+  networkstatus_voter_info_t *voter;
+
+  tor_assert(ns);
+  tor_assert(tokens);
+  /* Can be only one of them else code flow. */
+  tor_assert(ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS);
+
+  if (ns->type == NS_TYPE_VOTE) {
+    voter = smartlist_get(ns->voters, 0);
+    tor_assert(voter);
+    voter_identity = hex_str(voter->identity_digest,
+                             sizeof(voter->identity_digest));
+  } else {
+    /* Consensus has multiple voters so no specific voter. */
+    voter_identity = "consensus";
+  }
+
+  /* We extract both, and on error everything is stopped because it means
+   * the vote is malformed for the shared random value(s). */
+  if (extract_one_srv(tokens, K_PREVIOUS_SRV, &ns->sr_info.previous_srv) < 0) {
+    log_warn(LD_DIR, "SR: Unable to parse previous SRV from %s",
+             voter_identity);
+    /* Maybe we have a chance with the current SRV so let's try it anyway. */
+  }
+  if (extract_one_srv(tokens, K_CURRENT_SRV, &ns->sr_info.current_srv) < 0) {
+    log_warn(LD_DIR, "SR: Unable to parse current SRV from %s",
+             voter_identity);
+  }
+}
+
+/** Parse a v3 networkstatus vote, opinion, or consensus (depending on
+ * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
+networkstatus_t *
+networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
+                                     networkstatus_type_t ns_type)
+{
+  smartlist_t *tokens = smartlist_new();
+  smartlist_t *rs_tokens = NULL, *footer_tokens = NULL;
+  networkstatus_voter_info_t *voter = NULL;
+  networkstatus_t *ns = NULL;
+  common_digests_t ns_digests;
+  uint8_t sha3_as_signed[DIGEST256_LEN];
+  const char *cert, *end_of_header, *end_of_footer, *s_dup = s;
+  directory_token_t *tok;
+  struct in_addr in;
+  int i, inorder, n_signatures = 0;
+  memarea_t *area = NULL, *rs_area = NULL;
+  consensus_flavor_t flav = FLAV_NS;
+  char *last_kwd=NULL;
+
+  tor_assert(s);
+
+  if (eos_out)
+    *eos_out = NULL;
+
+  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
+    log_warn(LD_DIR, "Unable to compute digest of network-status");
+    goto err;
+  }
+
+  area = memarea_new();
+  end_of_header = find_start_of_next_routerstatus(s);
+  if (tokenize_string(area, s, end_of_header, tokens,
+                      (ns_type == NS_TYPE_CONSENSUS) ?
+                      networkstatus_consensus_token_table :
+                      networkstatus_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing network-status header");
+    goto err;
+  }
+
+  ns = tor_malloc_zero(sizeof(networkstatus_t));
+  memcpy(&ns->digests, &ns_digests, sizeof(ns_digests));
+  memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed));
+
+  tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION);
+  tor_assert(tok);
+  if (tok->n_args > 1) {
+    int flavor = networkstatus_parse_flavor_name(tok->args[1]);
+    if (flavor < 0) {
+      log_warn(LD_DIR, "Can't parse document with unknown flavor %s",
+               escaped(tok->args[1]));
+      goto err;
+    }
+    ns->flavor = flav = flavor;
+  }
+  if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) {
+    log_warn(LD_DIR, "Flavor found on non-consensus networkstatus.");
+    goto err;
+  }
+
+  if (ns_type != NS_TYPE_CONSENSUS) {
+    const char *end_of_cert = NULL;
+    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
+      goto err;
+    ++cert;
+    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
+    if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
+      goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_VOTE_STATUS);
+  tor_assert(tok->n_args);
+  if (!strcmp(tok->args[0], "vote")) {
+    ns->type = NS_TYPE_VOTE;
+  } else if (!strcmp(tok->args[0], "consensus")) {
+    ns->type = NS_TYPE_CONSENSUS;
+  } else if (!strcmp(tok->args[0], "opinion")) {
+    ns->type = NS_TYPE_OPINION;
+  } else {
+    log_warn(LD_DIR, "Unrecognized vote status %s in network-status",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  if (ns_type != ns->type) {
+    log_warn(LD_DIR, "Got the wrong kind of v3 networkstatus.");
+    goto err;
+  }
+
+  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) {
+    tok = find_by_keyword(tokens, K_PUBLISHED);
+    if (parse_iso_time(tok->args[0], &ns->published))
+      goto err;
+
+    ns->supported_methods = smartlist_new();
+    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHODS);
+    if (tok) {
+      for (i=0; i < tok->n_args; ++i)
+        smartlist_add_strdup(ns->supported_methods, tok->args[i]);
+    } else {
+      smartlist_add_strdup(ns->supported_methods, "1");
+    }
+  } else {
+    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHOD);
+    if (tok) {
+      int num_ok;
+      ns->consensus_method = (int)tor_parse_long(tok->args[0], 10, 1, INT_MAX,
+                                                 &num_ok, NULL);
+      if (!num_ok)
+        goto err;
+    } else {
+      ns->consensus_method = 1;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_CLIENT_PROTOCOLS)))
+    ns->recommended_client_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_RELAY_PROTOCOLS)))
+    ns->recommended_relay_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_CLIENT_PROTOCOLS)))
+    ns->required_client_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_RELAY_PROTOCOLS)))
+    ns->required_relay_protocols = tor_strdup(tok->args[0]);
+
+  tok = find_by_keyword(tokens, K_VALID_AFTER);
+  if (parse_iso_time(tok->args[0], &ns->valid_after))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
+  if (parse_iso_time(tok->args[0], &ns->fresh_until))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_VALID_UNTIL);
+  if (parse_iso_time(tok->args[0], &ns->valid_until))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_VOTING_DELAY);
+  tor_assert(tok->n_args >= 2);
+  {
+    int ok;
+    ns->vote_seconds =
+      (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &ok, NULL);
+    if (!ok)
+      goto err;
+    ns->dist_seconds =
+      (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL);
+    if (!ok)
+      goto err;
+  }
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) {
+    log_warn(LD_DIR, "Vote/consensus freshness interval is too short");
+    goto err;
+  }
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) {
+    log_warn(LD_DIR, "Vote/consensus liveness interval is too short");
+    goto err;
+  }
+  if (ns->vote_seconds < MIN_VOTE_SECONDS) {
+    log_warn(LD_DIR, "Vote seconds is too short");
+    goto err;
+  }
+  if (ns->dist_seconds < MIN_DIST_SECONDS) {
+    log_warn(LD_DIR, "Dist seconds is too short");
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) {
+    ns->client_versions = tor_strdup(tok->args[0]);
+  }
+  if ((tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS))) {
+    ns->server_versions = tor_strdup(tok->args[0]);
+  }
+
+  {
+    smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE);
+    ns->package_lines = smartlist_new();
+    if (package_lst) {
+      SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
+                    smartlist_add_strdup(ns->package_lines, t->args[0]));
+    }
+    smartlist_free(package_lst);
+  }
+
+  tok = find_by_keyword(tokens, K_KNOWN_FLAGS);
+  ns->known_flags = smartlist_new();
+  inorder = 1;
+  for (i = 0; i < tok->n_args; ++i) {
+    smartlist_add_strdup(ns->known_flags, tok->args[i]);
+    if (i>0 && strcmp(tok->args[i-1], tok->args[i])>= 0) {
+      log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
+      inorder = 0;
+    }
+  }
+  if (!inorder) {
+    log_warn(LD_DIR, "known-flags not in order");
+    goto err;
+  }
+  if (ns->type != NS_TYPE_CONSENSUS &&
+      smartlist_len(ns->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) {
+    /* If we allowed more than 64 flags in votes, then parsing them would make
+     * us invoke undefined behavior whenever we used 1<<flagnum to do a
+     * bit-shift. This is only for votes and opinions: consensus users don't
+     * care about flags they don't recognize, and so don't build a bitfield
+     * for them. */
+    log_warn(LD_DIR, "Too many known-flags in consensus vote or opinion");
+    goto err;
+  }
+
+  tok = find_opt_by_keyword(tokens, K_PARAMS);
+  if (tok) {
+    int any_dups = 0;
+    inorder = 1;
+    ns->net_params = smartlist_new();
+    for (i = 0; i < tok->n_args; ++i) {
+      int ok=0;
+      char *eq = strchr(tok->args[i], '=');
+      size_t eq_pos;
+      if (!eq) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      eq_pos = eq-tok->args[i];
+      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      if (i > 0 && strcmp(tok->args[i-1], tok->args[i]) >= 0) {
+        log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
+        inorder = 0;
+      }
+      if (last_kwd && eq_pos == strlen(last_kwd) &&
+          fast_memeq(last_kwd, tok->args[i], eq_pos)) {
+        log_warn(LD_DIR, "Duplicate value for %s parameter",
+                 escaped(tok->args[i]));
+        any_dups = 1;
+      }
+      tor_free(last_kwd);
+      last_kwd = tor_strndup(tok->args[i], eq_pos);
+      smartlist_add_strdup(ns->net_params, tok->args[i]);
+    }
+    if (!inorder) {
+      log_warn(LD_DIR, "params not in order");
+      goto err;
+    }
+    if (any_dups) {
+      log_warn(LD_DIR, "Duplicate in parameters");
+      goto err;
+    }
+  }
+
+  ns->voters = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    tok = _tok;
+    if (tok->tp == K_DIR_SOURCE) {
+      tor_assert(tok->n_args >= 6);
+
+      if (voter)
+        smartlist_add(ns->voters, voter);
+      voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+      voter->sigs = smartlist_new();
+      if (ns->type != NS_TYPE_CONSENSUS)
+        memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN);
+
+      voter->nickname = tor_strdup(tok->args[0]);
+      if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
+          base16_decode(voter->identity_digest, sizeof(voter->identity_digest),
+                        tok->args[1], HEX_DIGEST_LEN)
+                        != sizeof(voter->identity_digest)) {
+        log_warn(LD_DIR, "Error decoding identity digest %s in "
+                 "network-status document.", escaped(tok->args[1]));
+        goto err;
+      }
+      if (ns->type != NS_TYPE_CONSENSUS &&
+          tor_memneq(ns->cert->cache_info.identity_digest,
+                 voter->identity_digest, DIGEST_LEN)) {
+        log_warn(LD_DIR,"Mismatch between identities in certificate and vote");
+        goto err;
+      }
+      if (ns->type != NS_TYPE_CONSENSUS) {
+        if (authority_cert_is_blacklisted(ns->cert)) {
+          log_warn(LD_DIR, "Rejecting vote signature made with blacklisted "
+                   "signing key %s",
+                   hex_str(ns->cert->signing_key_digest, DIGEST_LEN));
+          goto err;
+        }
+      }
+      voter->address = tor_strdup(tok->args[2]);
+      if (!tor_inet_aton(tok->args[3], &in)) {
+        log_warn(LD_DIR, "Error decoding IP address %s in network-status.",
+                 escaped(tok->args[3]));
+        goto err;
+      }
+      voter->addr = ntohl(in.s_addr);
+      int ok;
+      voter->dir_port = (uint16_t)
+        tor_parse_long(tok->args[4], 10, 0, 65535, &ok, NULL);
+      if (!ok)
+        goto err;
+      voter->or_port = (uint16_t)
+        tor_parse_long(tok->args[5], 10, 0, 65535, &ok, NULL);
+      if (!ok)
+        goto err;
+    } else if (tok->tp == K_CONTACT) {
+      if (!voter || voter->contact) {
+        log_warn(LD_DIR, "contact element is out of place.");
+        goto err;
+      }
+      voter->contact = tor_strdup(tok->args[0]);
+    } else if (tok->tp == K_VOTE_DIGEST) {
+      tor_assert(ns->type == NS_TYPE_CONSENSUS);
+      tor_assert(tok->n_args >= 1);
+      if (!voter || ! tor_digest_is_zero(voter->vote_digest)) {
+        log_warn(LD_DIR, "vote-digest element is out of place.");
+        goto err;
+      }
+      if (strlen(tok->args[0]) != HEX_DIGEST_LEN ||
+        base16_decode(voter->vote_digest, sizeof(voter->vote_digest),
+                      tok->args[0], HEX_DIGEST_LEN)
+                      != sizeof(voter->vote_digest)) {
+        log_warn(LD_DIR, "Error decoding vote digest %s in "
+                 "network-status consensus.", escaped(tok->args[0]));
+        goto err;
+      }
+    }
+  } SMARTLIST_FOREACH_END(_tok);
+  if (voter) {
+    smartlist_add(ns->voters, voter);
+    voter = NULL;
+  }
+  if (smartlist_len(ns->voters) == 0) {
+    log_warn(LD_DIR, "Missing dir-source elements in a networkstatus.");
+    goto err;
+  } else if (ns->type != NS_TYPE_CONSENSUS && smartlist_len(ns->voters) != 1) {
+    log_warn(LD_DIR, "Too many dir-source elements in a vote networkstatus.");
+    goto err;
+  }
+
+  if (ns->type != NS_TYPE_CONSENSUS &&
+      (tok = find_opt_by_keyword(tokens, K_LEGACY_DIR_KEY))) {
+    int bad = 1;
+    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
+      networkstatus_voter_info_t *voter_0 = smartlist_get(ns->voters, 0);
+      if (base16_decode(voter_0->legacy_id_digest, DIGEST_LEN,
+                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN)
+        bad = 1;
+      else
+        bad = 0;
+    }
+    if (bad) {
+      log_warn(LD_DIR, "Invalid legacy key digest %s on vote.",
+               escaped(tok->args[0]));
+    }
+  }
+
+  /* If this is a vote document, check if information about the shared
+     randomness protocol is included, and extract it. */
+  if (ns->type == NS_TYPE_VOTE) {
+    dirvote_parse_sr_commits(ns, tokens);
+  }
+  /* For both a vote and consensus, extract the shared random values. */
+  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) {
+    extract_shared_random_srvs(ns, tokens);
+  }
+
+  /* Parse routerstatus lines. */
+  rs_tokens = smartlist_new();
+  rs_area = memarea_new();
+  s = end_of_header;
+  ns->routerstatus_list = smartlist_new();
+
+  while (!strcmpstart(s, "r ")) {
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
+                                               rs, 0, 0)) {
+        smartlist_add(ns->routerstatus_list, rs);
+      } else {
+        vote_routerstatus_free(rs);
+      }
+    } else {
+      routerstatus_t *rs;
+      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
+                                                     NULL, NULL,
+                                                     ns->consensus_method,
+                                                     flav))) {
+        /* Use exponential-backoff scheduling when downloading microdescs */
+        smartlist_add(ns->routerstatus_list, rs);
+      }
+    }
+  }
+  for (i = 1; i < smartlist_len(ns->routerstatus_list); ++i) {
+    routerstatus_t *rs1, *rs2;
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      vote_routerstatus_t *a = smartlist_get(ns->routerstatus_list, i-1);
+      vote_routerstatus_t *b = smartlist_get(ns->routerstatus_list, i);
+      rs1 = &a->status; rs2 = &b->status;
+    } else {
+      rs1 = smartlist_get(ns->routerstatus_list, i-1);
+      rs2 = smartlist_get(ns->routerstatus_list, i);
+    }
+    if (fast_memcmp(rs1->identity_digest, rs2->identity_digest, DIGEST_LEN)
+        >= 0) {
+      log_warn(LD_DIR, "Networkstatus entries not sorted by identity digest");
+      goto err;
+    }
+  }
+  if (ns_type != NS_TYPE_CONSENSUS) {
+    digest256map_t *ed_id_map = digest256map_new();
+    SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
+                            vrs) {
+      if (! vrs->has_ed25519_listing ||
+          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
+        continue;
+      if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
+        log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
+                 "unique");
+        digest256map_free(ed_id_map, NULL);
+        goto err;
+      }
+      digest256map_set(ed_id_map, vrs->ed25519_id, (void*)1);
+    } SMARTLIST_FOREACH_END(vrs);
+    digest256map_free(ed_id_map, NULL);
+  }
+
+  /* Parse footer; check signature. */
+  footer_tokens = smartlist_new();
+  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
+    ++end_of_footer;
+  else
+    end_of_footer = s + strlen(s);
+  if (tokenize_string(area,s, end_of_footer, footer_tokens,
+                      networkstatus_vote_footer_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
+    goto err;
+  }
+
+  {
+    int found_sig = 0;
+    SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
+      tok = _tok;
+      if (tok->tp == K_DIRECTORY_SIGNATURE)
+        found_sig = 1;
+      else if (found_sig) {
+        log_warn(LD_DIR, "Extraneous token after first directory-signature");
+        goto err;
+      }
+    } SMARTLIST_FOREACH_END(_tok);
+  }
+
+  if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) {
+    if (tok != smartlist_get(footer_tokens, 0)) {
+      log_warn(LD_DIR, "Misplaced directory-footer token");
+      goto err;
+    }
+  }
+
+  tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS);
+  if (tok) {
+    ns->weight_params = smartlist_new();
+    for (i = 0; i < tok->n_args; ++i) {
+      int ok=0;
+      char *eq = strchr(tok->args[i], '=');
+      if (!eq) {
+        log_warn(LD_DIR, "Bad element '%s' in weight params",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      smartlist_add_strdup(ns->weight_params, tok->args[i]);
+    }
+  }
+
+  SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
+    char declared_identity[DIGEST_LEN];
+    networkstatus_voter_info_t *v;
+    document_signature_t *sig;
+    const char *id_hexdigest = NULL;
+    const char *sk_hexdigest = NULL;
+    digest_algorithm_t alg = DIGEST_SHA1;
+    tok = _tok;
+    if (tok->tp != K_DIRECTORY_SIGNATURE)
+      continue;
+    tor_assert(tok->n_args >= 2);
+    if (tok->n_args == 2) {
+      id_hexdigest = tok->args[0];
+      sk_hexdigest = tok->args[1];
+    } else {
+      const char *algname = tok->args[0];
+      int a;
+      id_hexdigest = tok->args[1];
+      sk_hexdigest = tok->args[2];
+      a = crypto_digest_algorithm_parse_name(algname);
+      if (a<0) {
+        log_warn(LD_DIR, "Unknown digest algorithm %s; skipping",
+                 escaped(algname));
+        continue;
+      }
+      alg = a;
+    }
+
+    if (!tok->object_type ||
+        strcmp(tok->object_type, "SIGNATURE") ||
+        tok->object_size < 128 || tok->object_size > 512) {
+      log_warn(LD_DIR, "Bad object type or length on directory-signature");
+      goto err;
+    }
+
+    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(declared_identity, sizeof(declared_identity),
+                      id_hexdigest, HEX_DIGEST_LEN)
+                      != sizeof(declared_identity)) {
+      log_warn(LD_DIR, "Error decoding declared identity %s in "
+               "network-status document.", escaped(id_hexdigest));
+      goto err;
+    }
+    if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) {
+      log_warn(LD_DIR, "ID on signature on network-status document does "
+               "not match any declared directory source.");
+      goto err;
+    }
+    sig = tor_malloc_zero(sizeof(document_signature_t));
+    memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN);
+    sig->alg = alg;
+    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest),
+                      sk_hexdigest, HEX_DIGEST_LEN)
+                      != sizeof(sig->signing_key_digest)) {
+      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
+               "network-status document.", escaped(sk_hexdigest));
+      tor_free(sig);
+      goto err;
+    }
+
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      if (tor_memneq(declared_identity, ns->cert->cache_info.identity_digest,
+                 DIGEST_LEN)) {
+        log_warn(LD_DIR, "Digest mismatch between declared and actual on "
+                 "network-status vote.");
+        tor_free(sig);
+        goto err;
+      }
+    }
+
+    if (networkstatus_get_voter_sig_by_alg(v, sig->alg)) {
+      /* We already parsed a vote with this algorithm from this voter. Use the
+         first one. */
+      log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus "
+             "that contains two signatures from the same voter with the same "
+             "algorithm. Ignoring the second signature.");
+      tor_free(sig);
+      continue;
+    }
+
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN,
+                                tok, ns->cert->signing_key, 0,
+                                "network-status document")) {
+        tor_free(sig);
+        goto err;
+      }
+      sig->good_signature = 1;
+    } else {
+      if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
+        tor_free(sig);
+        goto err;
+      }
+      sig->signature = tor_memdup(tok->object_body, tok->object_size);
+      sig->signature_len = (int) tok->object_size;
+    }
+    smartlist_add(v->sigs, sig);
+
+    ++n_signatures;
+  } SMARTLIST_FOREACH_END(_tok);
+
+  if (! n_signatures) {
+    log_warn(LD_DIR, "No signatures on networkstatus document.");
+    goto err;
+  } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) {
+    log_warn(LD_DIR, "Received more than one signature on a "
+             "network-status vote.");
+    goto err;
+  }
+
+  if (eos_out)
+    *eos_out = end_of_footer;
+
+  goto done;
+ err:
+  dump_desc(s_dup, "v3 networkstatus");
+  networkstatus_vote_free(ns);
+  ns = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (voter) {
+    if (voter->sigs) {
+      SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                        document_signature_free(sig));
+      smartlist_free(voter->sigs);
+    }
+    tor_free(voter->nickname);
+    tor_free(voter->address);
+    tor_free(voter->contact);
+    tor_free(voter);
+  }
+  if (rs_tokens) {
+    SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(rs_tokens);
+  }
+  if (footer_tokens) {
+    SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(footer_tokens);
+  }
+  if (area) {
+    DUMP_AREA(area, "v3 networkstatus");
+    memarea_drop_all(area);
+  }
+  if (rs_area)
+    memarea_drop_all(rs_area);
+  tor_free(last_kwd);
+
+  return ns;
+}
+
+/** Return the common_digests_t that holds the digests of the
+ * <b>flavor_name</b>-flavored networkstatus according to the detached
+ * signatures document <b>sigs</b>, allocating a new common_digests_t as
+ * needed. */
+static common_digests_t *
+detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name)
+{
+  common_digests_t *d = strmap_get(sigs->digests, flavor_name);
+  if (!d) {
+    d = tor_malloc_zero(sizeof(common_digests_t));
+    strmap_set(sigs->digests, flavor_name, d);
+  }
+  return d;
+}
+
+/** Return the list of signatures of the <b>flavor_name</b>-flavored
+ * networkstatus according to the detached signatures document <b>sigs</b>,
+ * allocating a new common_digests_t as needed. */
+static smartlist_t *
+detached_get_signatures(ns_detached_signatures_t *sigs,
+                        const char *flavor_name)
+{
+  smartlist_t *sl = strmap_get(sigs->signatures, flavor_name);
+  if (!sl) {
+    sl = smartlist_new();
+    strmap_set(sigs->signatures, flavor_name, sl);
+  }
+  return sl;
+}
+
+/** Parse a detached v3 networkstatus signature document between <b>s</b> and
+ * <b>eos</b> and return the result.  Return -1 on failure. */
+ns_detached_signatures_t *
+networkstatus_parse_detached_signatures(const char *s, const char *eos)
+{
+  /* XXXX there is too much duplicate shared between this function and
+   * networkstatus_parse_vote_from_string(). */
+  directory_token_t *tok;
+  memarea_t *area = NULL;
+  common_digests_t *digests;
+
+  smartlist_t *tokens = smartlist_new();
+  ns_detached_signatures_t *sigs =
+    tor_malloc_zero(sizeof(ns_detached_signatures_t));
+  sigs->digests = strmap_new();
+  sigs->signatures = strmap_new();
+
+  if (!eos)
+    eos = s + strlen(s);
+
+  area = memarea_new();
+  if (tokenize_string(area,s, eos, tokens,
+                      networkstatus_detached_signature_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing detached networkstatus signatures");
+    goto err;
+  }
+
+  /* Grab all the digest-like tokens. */
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    const char *algname;
+    digest_algorithm_t alg;
+    const char *flavor;
+    const char *hexdigest;
+    size_t expected_length, digest_length;
+
+    tok = _tok;
+
+    if (tok->tp == K_CONSENSUS_DIGEST) {
+      algname = "sha1";
+      alg = DIGEST_SHA1;
+      flavor = "ns";
+      hexdigest = tok->args[0];
+    } else if (tok->tp == K_ADDITIONAL_DIGEST) {
+      int a = crypto_digest_algorithm_parse_name(tok->args[1]);
+      if (a<0) {
+        log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]);
+        continue;
+      }
+      alg = (digest_algorithm_t) a;
+      flavor = tok->args[0];
+      algname = tok->args[1];
+      hexdigest = tok->args[2];
+    } else {
+      continue;
+    }
+
+    digest_length = crypto_digest_algorithm_get_length(alg);
+    expected_length = digest_length * 2; /* hex encoding */
+
+    if (strlen(hexdigest) != expected_length) {
+      log_warn(LD_DIR, "Wrong length on consensus-digest in detached "
+               "networkstatus signatures");
+      goto err;
+    }
+    digests = detached_get_digests(sigs, flavor);
+    tor_assert(digests);
+    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
+      log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
+               "signatures document", flavor, algname);
+      continue;
+    }
+    if (base16_decode(digests->d[alg], digest_length,
+                      hexdigest, strlen(hexdigest)) != (int) digest_length) {
+      log_warn(LD_DIR, "Bad encoding on consensus-digest in detached "
+               "networkstatus signatures");
+      goto err;
+    }
+  } SMARTLIST_FOREACH_END(_tok);
+
+  tok = find_by_keyword(tokens, K_VALID_AFTER);
+  if (parse_iso_time(tok->args[0], &sigs->valid_after)) {
+    log_warn(LD_DIR, "Bad valid-after in detached networkstatus signatures");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
+  if (parse_iso_time(tok->args[0], &sigs->fresh_until)) {
+    log_warn(LD_DIR, "Bad fresh-until in detached networkstatus signatures");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_VALID_UNTIL);
+  if (parse_iso_time(tok->args[0], &sigs->valid_until)) {
+    log_warn(LD_DIR, "Bad valid-until in detached networkstatus signatures");
+    goto err;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    const char *id_hexdigest;
+    const char *sk_hexdigest;
+    const char *algname;
+    const char *flavor;
+    digest_algorithm_t alg;
+
+    char id_digest[DIGEST_LEN];
+    char sk_digest[DIGEST_LEN];
+    smartlist_t *siglist;
+    document_signature_t *sig;
+    int is_duplicate;
+
+    tok = _tok;
+    if (tok->tp == K_DIRECTORY_SIGNATURE) {
+      tor_assert(tok->n_args >= 2);
+      flavor = "ns";
+      algname = "sha1";
+      id_hexdigest = tok->args[0];
+      sk_hexdigest = tok->args[1];
+    } else if (tok->tp == K_ADDITIONAL_SIGNATURE) {
+      tor_assert(tok->n_args >= 4);
+      flavor = tok->args[0];
+      algname = tok->args[1];
+      id_hexdigest = tok->args[2];
+      sk_hexdigest = tok->args[3];
+    } else {
+      continue;
+    }
+
+    {
+      int a = crypto_digest_algorithm_parse_name(algname);
+      if (a<0) {
+        log_warn(LD_DIR, "Unrecognized algorithm name %s", algname);
+        continue;
+      }
+      alg = (digest_algorithm_t) a;
+    }
+
+    if (!tok->object_type ||
+        strcmp(tok->object_type, "SIGNATURE") ||
+        tok->object_size < 128 || tok->object_size > 512) {
+      log_warn(LD_DIR, "Bad object type or length on directory-signature");
+      goto err;
+    }
+
+    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(id_digest, sizeof(id_digest),
+                      id_hexdigest, HEX_DIGEST_LEN) != sizeof(id_digest)) {
+      log_warn(LD_DIR, "Error decoding declared identity %s in "
+               "network-status vote.", escaped(id_hexdigest));
+      goto err;
+    }
+    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(sk_digest, sizeof(sk_digest),
+                      sk_hexdigest, HEX_DIGEST_LEN) != sizeof(sk_digest)) {
+      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
+               "network-status vote.", escaped(sk_hexdigest));
+      goto err;
+    }
+
+    siglist = detached_get_signatures(sigs, flavor);
+    is_duplicate = 0;
+    SMARTLIST_FOREACH(siglist, document_signature_t *, dsig, {
+      if (dsig->alg == alg &&
+          tor_memeq(id_digest, dsig->identity_digest, DIGEST_LEN) &&
+          tor_memeq(sk_digest, dsig->signing_key_digest, DIGEST_LEN)) {
+        is_duplicate = 1;
+      }
+    });
+    if (is_duplicate) {
+      log_warn(LD_DIR, "Two signatures with identical keys and algorithm "
+               "found.");
+      continue;
+    }
+
+    sig = tor_malloc_zero(sizeof(document_signature_t));
+    sig->alg = alg;
+    memcpy(sig->identity_digest, id_digest, DIGEST_LEN);
+    memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN);
+    if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
+      tor_free(sig);
+      goto err;
+    }
+    sig->signature = tor_memdup(tok->object_body, tok->object_size);
+    sig->signature_len = (int) tok->object_size;
+
+    smartlist_add(siglist, sig);
+  } SMARTLIST_FOREACH_END(_tok);
+
+  goto done;
+ err:
+  ns_detached_signatures_free(sigs);
+  sigs = NULL;
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "detached signatures");
+    memarea_drop_all(area);
+  }
+  return sigs;
+}
+
+/** Parse the addr policy in the string <b>s</b> and return it.  If
+ * assume_action is nonnegative, then insert its action (ADDR_POLICY_ACCEPT or
+ * ADDR_POLICY_REJECT) for items that specify no action.
+ *
+ * Returns NULL on policy errors.
+ *
+ * Set *<b>malformed_list</b> to true if the entire policy list should be
+ * discarded. Otherwise, set it to false, and only this item should be ignored
+ * on error - the rest of the policy list can continue to be processed and
+ * used.
+ *
+ * The addr_policy_t returned by this function can have its address set to
+ * AF_UNSPEC for '*'.  Use policy_expand_unspec() to turn this into a pair
+ * of AF_INET and AF_INET6 items.
+ */
+MOCK_IMPL(addr_policy_t *,
+router_parse_addr_policy_item_from_string,(const char *s, int assume_action,
+                                           int *malformed_list))
+{
+  directory_token_t *tok = NULL;
+  const char *cp, *eos;
+  /* Longest possible policy is
+   * "accept6 [ffff:ffff:..255]/128:10000-65535",
+   * which contains a max-length IPv6 address, plus 26 characters.
+   * But note that there can be an arbitrary amount of space between the
+   * accept and the address:mask/port element.
+   * We don't need to multiply TOR_ADDR_BUF_LEN by 2, as there is only one
+   * IPv6 address. But making the buffer shorter might cause valid long lines,
+   * which parsed in previous versions, to fail to parse in new versions.
+   * (These lines would have to have excessive amounts of whitespace.) */
+  char line[TOR_ADDR_BUF_LEN*2 + 32];
+  addr_policy_t *r;
+  memarea_t *area = NULL;
+
+  tor_assert(malformed_list);
+  *malformed_list = 0;
+
+  s = eat_whitespace(s);
+  /* We can only do assume_action on []-quoted IPv6, as "a" (accept)
+   * and ":" (port separator) are ambiguous */
+  if ((*s == '*' || *s == '[' || TOR_ISDIGIT(*s)) && assume_action >= 0) {
+    if (tor_snprintf(line, sizeof(line), "%s %s",
+               assume_action == ADDR_POLICY_ACCEPT?"accept":"reject", s)<0) {
+      log_warn(LD_DIR, "Policy %s is too long.", escaped(s));
+      return NULL;
+    }
+    cp = line;
+    tor_strlower(line);
+  } else { /* assume an already well-formed address policy line */
+    cp = s;
+  }
+
+  eos = cp + strlen(cp);
+  area = memarea_new();
+  tok = get_next_token(area, &cp, eos, routerdesc_token_table);
+  if (tok->tp == ERR_) {
+    log_warn(LD_DIR, "Error reading address policy: %s", tok->error);
+    goto err;
+  }
+  if (tok->tp != K_ACCEPT && tok->tp != K_ACCEPT6 &&
+      tok->tp != K_REJECT && tok->tp != K_REJECT6) {
+    log_warn(LD_DIR, "Expected 'accept' or 'reject'.");
+    goto err;
+  }
+
+  /* Use the extended interpretation of accept/reject *,
+   * expanding it into an IPv4 wildcard and an IPv6 wildcard.
+   * Also permit *4 and *6 for IPv4 and IPv6 only wildcards. */
+  r = router_parse_addr_policy(tok, TAPMP_EXTENDED_STAR);
+  if (!r) {
+    goto err;
+  }
+
+  /* Ensure that accept6/reject6 fields are followed by IPv6 addresses.
+   * AF_UNSPEC addresses are only permitted on the accept/reject field type.
+   * Unlike descriptors, torrcs exit policy accept/reject can be followed by
+   * either an IPv4 or IPv6 address. */
+  if ((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
+       tor_addr_family(&r->addr) != AF_INET6) {
+    /* This is a non-fatal error, just ignore this one entry. */
+    *malformed_list = 0;
+    log_warn(LD_DIR, "IPv4 address '%s' with accept6/reject6 field type in "
+             "exit policy. Ignoring, but continuing to parse rules. (Use "
+             "accept/reject with IPv4 addresses.)",
+             tok->n_args == 1 ? tok->args[0] : "");
+    addr_policy_free(r);
+    r = NULL;
+    goto done;
+  }
+
+  goto done;
+ err:
+  *malformed_list = 1;
+  r = NULL;
+ done:
+  token_clear(tok);
+  if (area) {
+    DUMP_AREA(area, "policy item");
+    memarea_drop_all(area);
+  }
+  return r;
+}
+
+/** Add an exit policy stored in the token <b>tok</b> to the router info in
+ * <b>router</b>.  Return 0 on success, -1 on failure. */
+static int
+router_add_exit_policy(routerinfo_t *router, directory_token_t *tok)
+{
+  addr_policy_t *newe;
+  /* Use the standard interpretation of accept/reject *, an IPv4 wildcard. */
+  newe = router_parse_addr_policy(tok, 0);
+  if (!newe)
+    return -1;
+  if (! router->exit_policy)
+    router->exit_policy = smartlist_new();
+
+  /* Ensure that in descriptors, accept/reject fields are followed by
+   * IPv4 addresses, and accept6/reject6 fields are followed by
+   * IPv6 addresses. Unlike torrcs, descriptor exit policies do not permit
+   * accept/reject followed by IPv6. */
+  if (((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
+       tor_addr_family(&newe->addr) == AF_INET)
+      ||
+      ((tok->tp == K_ACCEPT || tok->tp == K_REJECT) &&
+       tor_addr_family(&newe->addr) == AF_INET6)) {
+    /* There's nothing the user can do about other relays' descriptors,
+     * so we don't provide usage advice here. */
+    log_warn(LD_DIR, "Mismatch between field type and address type in exit "
+             "policy '%s'. Discarding entire router descriptor.",
+             tok->n_args == 1 ? tok->args[0] : "");
+    addr_policy_free(newe);
+    return -1;
+  }
+
+  smartlist_add(router->exit_policy, newe);
+
+  return 0;
+}
+
+/** Given a K_ACCEPT[6] or K_REJECT[6] token and a router, create and return
+ * a new exit_policy_t corresponding to the token. If TAPMP_EXTENDED_STAR
+ * is set in fmt_flags, K_ACCEPT6 and K_REJECT6 tokens followed by *
+ * expand to IPv6-only policies, otherwise they expand to IPv4 and IPv6
+ * policies */
+static addr_policy_t *
+router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags)
+{
+  addr_policy_t newe;
+  char *arg;
+
+  tor_assert(tok->tp == K_REJECT || tok->tp == K_REJECT6 ||
+             tok->tp == K_ACCEPT || tok->tp == K_ACCEPT6);
+
+  if (tok->n_args != 1)
+    return NULL;
+  arg = tok->args[0];
+
+  if (!strcmpstart(arg,"private"))
+    return router_parse_addr_policy_private(tok);
+
+  memset(&newe, 0, sizeof(newe));
+
+  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
+    newe.policy_type = ADDR_POLICY_REJECT;
+  else
+    newe.policy_type = ADDR_POLICY_ACCEPT;
+
+  /* accept6/reject6 * produces an IPv6 wildcard address only.
+   * (accept/reject * produces rules for IPv4 and IPv6 wildcard addresses.) */
+  if ((fmt_flags & TAPMP_EXTENDED_STAR)
+      && (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6)) {
+    fmt_flags |= TAPMP_STAR_IPV6_ONLY;
+  }
+
+  if (tor_addr_parse_mask_ports(arg, fmt_flags, &newe.addr, &newe.maskbits,
+                                &newe.prt_min, &newe.prt_max) < 0) {
+    log_warn(LD_DIR,"Couldn't parse line %s. Dropping", escaped(arg));
+    return NULL;
+  }
+
+  return addr_policy_get_canonical_entry(&newe);
+}
+
+/** Parse an exit policy line of the format "accept[6]/reject[6] private:...".
+ * This didn't exist until Tor 0.1.1.15, so nobody should generate it in
+ * router descriptors until earlier versions are obsolete.
+ *
+ * accept/reject and accept6/reject6 private all produce rules for both
+ * IPv4 and IPv6 addresses.
+ */
+static addr_policy_t *
+router_parse_addr_policy_private(directory_token_t *tok)
+{
+  const char *arg;
+  uint16_t port_min, port_max;
+  addr_policy_t result;
+
+  arg = tok->args[0];
+  if (strcmpstart(arg, "private"))
+    return NULL;
+
+  arg += strlen("private");
+  arg = (char*) eat_whitespace(arg);
+  if (!arg || *arg != ':')
+    return NULL;
+
+  if (parse_port_range(arg+1, &port_min, &port_max)<0)
+    return NULL;
+
+  memset(&result, 0, sizeof(result));
+  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
+    result.policy_type = ADDR_POLICY_REJECT;
+  else
+    result.policy_type = ADDR_POLICY_ACCEPT;
+  result.is_private = 1;
+  result.prt_min = port_min;
+  result.prt_max = port_max;
+
+  if (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) {
+    log_warn(LD_GENERAL,
+             "'%s' expands into rules which apply to all private IPv4 and "
+             "IPv6 addresses. (Use accept/reject private:* for IPv4 and "
+             "IPv6.)", tok->n_args == 1 ? tok->args[0] : "");
+  }
+
+  return addr_policy_get_canonical_entry(&result);
+}
+
+/** Log and exit if <b>t</b> is malformed */
+void
+assert_addr_policy_ok(smartlist_t *lst)
+{
+  if (!lst) return;
+  SMARTLIST_FOREACH(lst, addr_policy_t *, t, {
+    tor_assert(t->policy_type == ADDR_POLICY_REJECT ||
+               t->policy_type == ADDR_POLICY_ACCEPT);
+    tor_assert(t->prt_min <= t->prt_max);
+  });
+}
+
+/** Return a newly allocated smartlist of all accept or reject tokens in
+ * <b>s</b>.
+ */
+static smartlist_t *
+find_all_exitpolicy(smartlist_t *s)
+{
+  smartlist_t *out = smartlist_new();
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+      if (t->tp == K_ACCEPT || t->tp == K_ACCEPT6 ||
+          t->tp == K_REJECT || t->tp == K_REJECT6)
+        smartlist_add(out,t));
+  return out;
+}
+
+/** Helper function for <b>router_get_hash_impl</b>: given <b>s</b>,
+ * <b>s_len</b>, <b>start_str</b>, <b>end_str</b>, and <b>end_c</b> with the
+ * same semantics as in that function, set *<b>start_out</b> (inclusive) and
+ * *<b>end_out</b> (exclusive) to the boundaries of the string to be hashed.
+ *
+ * Return 0 on success and -1 on failure.
+ */
+static int
+router_get_hash_impl_helper(const char *s, size_t s_len,
+                            const char *start_str,
+                            const char *end_str, char end_c,
+                            int log_severity,
+                            const char **start_out, const char **end_out)
+{
+  const char *start, *end;
+  start = tor_memstr(s, s_len, start_str);
+  if (!start) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find start of hashed material \"%s\"",start_str);
+    return -1;
+  }
+  if (start != s && *(start-1) != '\n') {
+    log_fn(log_severity,LD_DIR,
+             "first occurrence of \"%s\" is not at the start of a line",
+             start_str);
+    return -1;
+  }
+  end = tor_memstr(start+strlen(start_str),
+                   s_len - (start-s) - strlen(start_str), end_str);
+  if (!end) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find end of hashed material \"%s\"",end_str);
+    return -1;
+  }
+  end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str));
+  if (!end) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find EOL");
+    return -1;
+  }
+  ++end;
+
+  *start_out = start;
+  *end_out = end;
+  return 0;
+}
+
+/** Compute the digest of the substring of <b>s</b> taken from the first
+ * occurrence of <b>start_str</b> through the first instance of c after the
+ * first subsequent occurrence of <b>end_str</b>; store the 20-byte or 32-byte
+ * result in <b>digest</b>; return 0 on success.
+ *
+ * If no such substring exists, return -1.
+ */
+static int
+router_get_hash_impl(const char *s, size_t s_len, char *digest,
+                     const char *start_str,
+                     const char *end_str, char end_c,
+                     digest_algorithm_t alg)
+{
+  const char *start=NULL, *end=NULL;
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
+                                  &start,&end)<0)
+    return -1;
+
+  return router_compute_hash_final(digest, start, end-start, alg);
+}
+
+/** Compute the digest of the <b>len</b>-byte directory object at
+ * <b>start</b>, using <b>alg</b>. Store the result in <b>digest</b>, which
+ * must be long enough to hold it. */
+MOCK_IMPL(STATIC int,
+router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg))
+{
+  if (alg == DIGEST_SHA1) {
+    if (crypto_digest(digest, start, len) < 0) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
+  } else {
+    if (crypto_digest256(digest, start, len, alg) < 0) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/** As router_get_hash_impl, but compute all hashes. */
+static int
+router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests,
+                       const char *start_str,
+                       const char *end_str, char end_c)
+{
+  const char *start=NULL, *end=NULL;
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
+                                  &start,&end)<0)
+    return -1;
+
+  if (crypto_common_digests(digests, start, end-start)) {
+    log_warn(LD_BUG,"couldn't compute digests");
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Assuming that s starts with a microdesc, return the start of the
+ * *NEXT* one.  Return NULL on "not found." */
+static const char *
+find_start_of_next_microdesc(const char *s, const char *eos)
+{
+  int started_with_annotations;
+  s = eat_whitespace_eos(s, eos);
+  if (!s)
+    return NULL;
+
+#define CHECK_LENGTH() STMT_BEGIN \
+    if (s+32 > eos)               \
+      return NULL;                \
+  STMT_END
+
+#define NEXT_LINE() STMT_BEGIN            \
+    s = memchr(s, '\n', eos-s);           \
+    if (!s || s+1 >= eos)                 \
+      return NULL;                        \
+    s++;                                  \
+  STMT_END
+
+  CHECK_LENGTH();
+
+  started_with_annotations = (*s == '@');
+
+  if (started_with_annotations) {
+    /* Start by advancing to the first non-annotation line. */
+    while (*s == '@')
+      NEXT_LINE();
+  }
+  CHECK_LENGTH();
+
+  /* Now we should be pointed at an onion-key line.  If we are, then skip
+   * it. */
+  if (!strcmpstart(s, "onion-key"))
+    NEXT_LINE();
+
+  /* Okay, now we're pointed at the first line of the microdescriptor which is
+     not an annotation or onion-key.  The next line that _is_ an annotation or
+     onion-key is the start of the next microdescriptor. */
+  while (s+32 < eos) {
+    if (*s == '@' || !strcmpstart(s, "onion-key"))
+      return s;
+    NEXT_LINE();
+  }
+  return NULL;
+
+#undef CHECK_LENGTH
+#undef NEXT_LINE
+}
+
+/** Parse as many microdescriptors as are found from the string starting at
+ * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
+ * annotations we recognize and ignore ones we don't.
+ *
+ * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
+ * descriptor in the body field of each microdesc_t.
+ *
+ * Return all newly parsed microdescriptors in a newly allocated
+ * smartlist_t. If <b>invalid_disgests_out</b> is provided, add a SHA256
+ * microdesc digest to it for every microdesc that we found to be badly
+ * formed. (This may cause duplicates) */
+smartlist_t *
+microdescs_parse_from_string(const char *s, const char *eos,
+                             int allow_annotations,
+                             saved_location_t where,
+                             smartlist_t *invalid_digests_out)
+{
+  smartlist_t *tokens;
+  smartlist_t *result;
+  microdesc_t *md = NULL;
+  memarea_t *area;
+  const char *start = s;
+  const char *start_of_next_microdesc;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+  const int copy_body = (where != SAVED_IN_CACHE);
+
+  directory_token_t *tok;
+
+  if (!eos)
+    eos = s + strlen(s);
+
+  s = eat_whitespace_eos(s, eos);
+  area = memarea_new();
+  result = smartlist_new();
+  tokens = smartlist_new();
+
+  while (s < eos) {
+    int okay = 0;
+
+    start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
+    if (!start_of_next_microdesc)
+      start_of_next_microdesc = eos;
+
+    md = tor_malloc_zero(sizeof(microdesc_t));
+    {
+      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
+                                  "onion-key");
+      const int no_onion_key = (cp == NULL);
+      if (no_onion_key) {
+        cp = s; /* So that we have *some* junk to put in the body */
+      }
+
+      md->bodylen = start_of_next_microdesc - cp;
+      md->saved_location = where;
+      if (copy_body)
+        md->body = tor_memdup_nulterm(cp, md->bodylen);
+      else
+        md->body = (char*)cp;
+      md->off = cp - start;
+      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+      if (no_onion_key) {
+        log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
+        goto next;
+      }
+    }
+
+    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                        microdesc_token_table, flags)) {
+      log_warn(LD_DIR, "Unparseable microdescriptor");
+      goto next;
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+      if (parse_iso_time(tok->args[0], &md->last_listed)) {
+        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+        goto next;
+      }
+    }
+
+    tok = find_by_keyword(tokens, K_ONION_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_DIR,
+               "Relay's onion key had invalid exponent.");
+      goto next;
+    }
+    md->onion_pkey = tok->key;
+    tok->key = NULL;
+
+    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+      curve25519_public_key_t k;
+      tor_assert(tok->n_args >= 1);
+      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
+        goto next;
+      }
+      md->onion_curve25519_pkey =
+        tor_memdup(&k, sizeof(curve25519_public_key_t));
+    }
+
+    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
+    if (id_lines) {
+      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
+        tor_assert(t->n_args >= 2);
+        if (!strcmp(t->args[0], "ed25519")) {
+          if (md->ed25519_identity_pkey) {
+            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+            smartlist_free(id_lines);
+            goto next;
+          }
+          ed25519_public_key_t k;
+          if (ed25519_public_from_base64(&k, t->args[1])<0) {
+            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+            smartlist_free(id_lines);
+            goto next;
+          }
+          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
+        }
+      } SMARTLIST_FOREACH_END(t);
+      smartlist_free(id_lines);
+    }
+
+    {
+      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+      if (a_lines) {
+        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
+        smartlist_free(a_lines);
+      }
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+      int i;
+      md->family = smartlist_new();
+      for (i=0;i<tok->n_args;++i) {
+        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+          log_warn(LD_DIR, "Illegal nickname %s in family line",
+                   escaped(tok->args[i]));
+          goto next;
+        }
+        smartlist_add_strdup(md->family, tok->args[i]);
+      }
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_P))) {
+      md->exit_policy = parse_short_policy(tok->args[0]);
+    }
+    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
+      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+    }
+
+    smartlist_add(result, md);
+    okay = 1;
+
+    md = NULL;
+  next:
+    if (! okay && invalid_digests_out) {
+      smartlist_add(invalid_digests_out,
+                    tor_memdup(md->digest, DIGEST256_LEN));
+    }
+    microdesc_free(md);
+    md = NULL;
+
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    memarea_clear(area);
+    smartlist_clear(tokens);
+    s = start_of_next_microdesc;
+  }
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+
+  return result;
+}
+
+/** Extract a Tor version from a <b>platform</b> line from a router
+ * descriptor, and place the result in <b>router_version</b>.
+ *
+ * Return 1 on success, -1 on parsing failure, and 0 if the
+ * platform line does not indicate some version of Tor.
+ *
+ * If <b>strict</b> is non-zero, finding any weird version components
+ * (like negative numbers) counts as a parsing failure.
+ */
+int
+tor_version_parse_platform(const char *platform,
+                           tor_version_t *router_version,
+                           int strict)
+{
+  char tmp[128];
+  char *s, *s2, *start;
+
+  if (strcmpstart(platform,"Tor ")) /* nonstandard Tor; say 0. */
+    return 0;
+
+  start = (char *)eat_whitespace(platform+3);
+  if (!*start) return -1;
+  s = (char *)find_whitespace(start); /* also finds '\0', which is fine */
+  s2 = (char*)eat_whitespace(s);
+  if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-"))
+    s = (char*)find_whitespace(s2);
+
+  if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */
+    return -1;
+  strlcpy(tmp, start, s-start+1);
+
+  if (tor_version_parse(tmp, router_version)<0) {
+    log_info(LD_DIR,"Router version '%s' unparseable.",tmp);
+    return -1;
+  }
+
+  if (strict) {
+    if (router_version->major < 0 ||
+        router_version->minor < 0 ||
+        router_version->micro < 0 ||
+        router_version->patchlevel < 0 ||
+        router_version->svn_revision < 0) {
+      return -1;
+    }
+  }
+
+  return 1;
+}
+
+/** Parse the Tor version of the platform string <b>platform</b>,
+ * and compare it to the version in <b>cutoff</b>. Return 1 if
+ * the router is at least as new as the cutoff, else return 0.
+ */
+int
+tor_version_as_new_as(const char *platform, const char *cutoff)
+{
+  tor_version_t cutoff_version, router_version;
+  int r;
+  tor_assert(platform);
+
+  if (tor_version_parse(cutoff, &cutoff_version)<0) {
+    log_warn(LD_BUG,"cutoff version '%s' unparseable.",cutoff);
+    return 0;
+  }
+
+  r = tor_version_parse_platform(platform, &router_version, 0);
+  if (r == 0) {
+    /* nonstandard Tor; be safe and say yes */
+    return 1;
+  } else if (r < 0) {
+    /* unparseable version; be safe and say yes. */
+    return 1;
+  }
+
+  /* Here's why we don't need to do any special handling for svn revisions:
+   * - If neither has an svn revision, we're fine.
+   * - If the router doesn't have an svn revision, we can't assume that it
+   *   is "at least" any svn revision, so we need to return 0.
+   * - If the target version doesn't have an svn revision, any svn revision
+   *   (or none at all) is good enough, so return 1.
+   * - If both target and router have an svn revision, we compare them.
+   */
+
+  return tor_version_compare(&router_version, &cutoff_version) >= 0;
+}
+
+/** Parse a tor version from <b>s</b>, and store the result in <b>out</b>.
+ * Return 0 on success, -1 on failure. */
+int
+tor_version_parse(const char *s, tor_version_t *out)
+{
+  char *eos=NULL;
+  const char *cp=NULL;
+  int ok = 1;
+  /* Format is:
+   *   "Tor " ? NUM dot NUM [ dot NUM [ ( pre | rc | dot ) NUM ] ] [ - tag ]
+   */
+  tor_assert(s);
+  tor_assert(out);
+
+  memset(out, 0, sizeof(tor_version_t));
+  out->status = VER_RELEASE;
+  if (!strcasecmpstart(s, "Tor "))
+    s += 4;
+
+  cp = s;
+
+#define NUMBER(m)                               \
+  do {                                          \
+    if (!cp || *cp < '0' || *cp > '9')          \
+      return -1;                                \
+    out->m = (int)tor_parse_uint64(cp, 10, 0, INT32_MAX, &ok, &eos);    \
+    if (!ok)                                    \
+      return -1;                                \
+    if (!eos || eos == cp)                      \
+      return -1;                                \
+    cp = eos;                                   \
+  } while (0)
+
+#define DOT()                                   \
+  do {                                          \
+    if (*cp != '.')                             \
+      return -1;                                \
+    ++cp;                                       \
+  } while (0)
+
+  NUMBER(major);
+  DOT();
+  NUMBER(minor);
+  if (*cp == 0)
+    return 0;
+  else if (*cp == '-')
+    goto status_tag;
+  DOT();
+  NUMBER(micro);
+
+  /* Get status */
+  if (*cp == 0) {
+    return 0;
+  } else if (*cp == '.') {
+    ++cp;
+  } else if (*cp == '-') {
+    goto status_tag;
+  } else if (0==strncmp(cp, "pre", 3)) {
+    out->status = VER_PRE;
+    cp += 3;
+  } else if (0==strncmp(cp, "rc", 2)) {
+    out->status = VER_RC;
+    cp += 2;
+  } else {
+    return -1;
+  }
+
+  NUMBER(patchlevel);
+
+ status_tag:
+  /* Get status tag. */
+  if (*cp == '-' || *cp == '.')
+    ++cp;
+  eos = (char*) find_whitespace(cp);
+  if (eos-cp >= (int)sizeof(out->status_tag))
+    strlcpy(out->status_tag, cp, sizeof(out->status_tag));
+  else {
+    memcpy(out->status_tag, cp, eos-cp);
+    out->status_tag[eos-cp] = 0;
+  }
+  cp = eat_whitespace(eos);
+
+  if (!strcmpstart(cp, "(r")) {
+    cp += 2;
+    out->svn_revision = (int) strtol(cp,&eos,10);
+  } else if (!strcmpstart(cp, "(git-")) {
+    char *close_paren = strchr(cp, ')');
+    int hexlen;
+    char digest[DIGEST_LEN];
+    if (! close_paren)
+      return -1;
+    cp += 5;
+    if (close_paren-cp > HEX_DIGEST_LEN)
+      return -1;
+    hexlen = (int)(close_paren-cp);
+    memwipe(digest, 0, sizeof(digest));
+    if ( hexlen == 0 || (hexlen % 2) == 1)
+      return -1;
+    if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2)
+      return -1;
+    memcpy(out->git_tag, digest, hexlen/2);
+    out->git_tag_len = hexlen/2;
+  }
+
+  return 0;
+#undef NUMBER
+#undef DOT
+}
+
+/** Compare two tor versions; Return <0 if a < b; 0 if a ==b, >0 if a >
+ * b. */
+int
+tor_version_compare(tor_version_t *a, tor_version_t *b)
+{
+  int i;
+  tor_assert(a);
+  tor_assert(b);
+
+  /* We take this approach to comparison to ensure the same (bogus!) behavior
+   * on all inputs as we would have seen before bug #21278 was fixed. The
+   * only important difference here is that this method doesn't cause
+   * a signed integer underflow.
+   */
+#define CMP(field) do {                               \
+    unsigned aval = (unsigned) a->field;              \
+    unsigned bval = (unsigned) b->field;              \
+    int result = (int) (aval - bval);                 \
+    if (result < 0)                                   \
+      return -1;                                      \
+    else if (result > 0)                              \
+      return 1;                                       \
+  } while (0)
+
+  CMP(major);
+  CMP(minor);
+  CMP(micro);
+  CMP(status);
+  CMP(patchlevel);
+  if ((i = strcmp(a->status_tag, b->status_tag)))
+     return i;
+  CMP(svn_revision);
+  CMP(git_tag_len);
+  if (a->git_tag_len)
+     return fast_memcmp(a->git_tag, b->git_tag, a->git_tag_len);
+  else
+     return 0;
+
+#undef CMP
+}
+
+/** Return true iff versions <b>a</b> and <b>b</b> belong to the same series.
+ */
+int
+tor_version_same_series(tor_version_t *a, tor_version_t *b)
+{
+  tor_assert(a);
+  tor_assert(b);
+  return ((a->major == b->major) &&
+          (a->minor == b->minor) &&
+          (a->micro == b->micro));
+}
+
+/** Helper: Given pointers to two strings describing tor versions, return -1
+ * if _a precedes _b, 1 if _b precedes _a, and 0 if they are equivalent.
+ * Used to sort a list of versions. */
+static int
+compare_tor_version_str_ptr_(const void **_a, const void **_b)
+{
+  const char *a = *_a, *b = *_b;
+  int ca, cb;
+  tor_version_t va, vb;
+  ca = tor_version_parse(a, &va);
+  cb = tor_version_parse(b, &vb);
+  /* If they both parse, compare them. */
+  if (!ca && !cb)
+    return tor_version_compare(&va,&vb);
+  /* If one parses, it comes first. */
+  if (!ca && cb)
+    return -1;
+  if (ca && !cb)
+    return 1;
+  /* If neither parses, compare strings.  Also, the directory server admin
+  ** needs to be smacked upside the head.  But Tor is tolerant and gentle. */
+  return strcmp(a,b);
+}
+
+/** Sort a list of string-representations of versions in ascending order. */
+void
+sort_version_list(smartlist_t *versions, int remove_duplicates)
+{
+  smartlist_sort(versions, compare_tor_version_str_ptr_);
+
+  if (remove_duplicates)
+    smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
+}
+
+/** Parse and validate the ASCII-encoded v2 descriptor in <b>desc</b>,
+ * write the parsed descriptor to the newly allocated *<b>parsed_out</b>, the
+ * binary descriptor ID of length DIGEST_LEN to <b>desc_id_out</b>, the
+ * encrypted introduction points to the newly allocated
+ * *<b>intro_points_encrypted_out</b>, their encrypted size to
+ * *<b>intro_points_encrypted_size_out</b>, the size of the encoded descriptor
+ * to *<b>encoded_size_out</b>, and a pointer to the possibly next
+ * descriptor to *<b>next_out</b>; return 0 for success (including validation)
+ * and -1 for failure.
+ *
+ * If <b>as_hsdir</b> is 1, we're parsing this as an HSDir, and we should
+ * be strict about time formats.
+ */
+int
+rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
+                                 char *desc_id_out,
+                                 char **intro_points_encrypted_out,
+                                 size_t *intro_points_encrypted_size_out,
+                                 size_t *encoded_size_out,
+                                 const char **next_out, const char *desc,
+                                 int as_hsdir)
+{
+  rend_service_descriptor_t *result =
+                            tor_malloc_zero(sizeof(rend_service_descriptor_t));
+  char desc_hash[DIGEST_LEN];
+  const char *eos;
+  smartlist_t *tokens = smartlist_new();
+  directory_token_t *tok;
+  char secret_id_part[DIGEST_LEN];
+  int i, version, num_ok=1;
+  smartlist_t *versions;
+  char public_key_hash[DIGEST_LEN];
+  char test_desc_id[DIGEST_LEN];
+  memarea_t *area = NULL;
+  const int strict_time_fmt = as_hsdir;
+
+  tor_assert(desc);
+  /* Check if desc starts correctly. */
+  if (strncmp(desc, "rendezvous-service-descriptor ",
+              strlen("rendezvous-service-descriptor "))) {
+    log_info(LD_REND, "Descriptor does not start correctly.");
+    goto err;
+  }
+  /* Compute descriptor hash for later validation. */
+  if (router_get_hash_impl(desc, strlen(desc), desc_hash,
+                           "rendezvous-service-descriptor ",
+                           "\nsignature", '\n', DIGEST_SHA1) < 0) {
+    log_warn(LD_REND, "Couldn't compute descriptor hash.");
+    goto err;
+  }
+  /* Determine end of string. */
+  eos = strstr(desc, "\nrendezvous-service-descriptor ");
+  if (!eos)
+    eos = desc + strlen(desc);
+  else
+    eos = eos + 1;
+  /* Check length. */
+  if (eos-desc > REND_DESC_MAX_SIZE) {
+    /* XXXX+ If we are parsing this descriptor as a server, this
+     * should be a protocol warning. */
+    log_warn(LD_REND, "Descriptor length is %d which exceeds "
+             "maximum rendezvous descriptor size of %d bytes.",
+             (int)(eos-desc), REND_DESC_MAX_SIZE);
+    goto err;
+  }
+  /* Tokenize descriptor. */
+  area = memarea_new();
+  if (tokenize_string(area, desc, eos, tokens, desc_token_table, 0)) {
+    log_warn(LD_REND, "Error tokenizing descriptor.");
+    goto err;
+  }
+  /* Set next to next descriptor, if available. */
+  *next_out = eos;
+  /* Set length of encoded descriptor. */
+  *encoded_size_out = eos - desc;
+  /* Check min allowed length of token list. */
+  if (smartlist_len(tokens) < 7) {
+    log_warn(LD_REND, "Impossibly short descriptor.");
+    goto err;
+  }
+  /* Parse base32-encoded descriptor ID. */
+  tok = find_by_keyword(tokens, R_RENDEZVOUS_SERVICE_DESCRIPTOR);
+  tor_assert(tok == smartlist_get(tokens, 0));
+  tor_assert(tok->n_args == 1);
+  if (!rend_valid_descriptor_id(tok->args[0])) {
+    log_warn(LD_REND, "Invalid descriptor ID: '%s'", tok->args[0]);
+    goto err;
+  }
+  if (base32_decode(desc_id_out, DIGEST_LEN,
+                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
+    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
+             tok->args[0]);
+    goto err;
+  }
+  /* Parse descriptor version. */
+  tok = find_by_keyword(tokens, R_VERSION);
+  tor_assert(tok->n_args == 1);
+  result->version =
+    (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &num_ok, NULL);
+  if (result->version != 2 || !num_ok) {
+    /* If it's <2, it shouldn't be under this format.  If the number
+     * is greater than 2, we bumped it because we broke backward
+     * compatibility.  See how version numbers in our other formats
+     * work. */
+    log_warn(LD_REND, "Unrecognized descriptor version: %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  /* Parse public key. */
+  tok = find_by_keyword(tokens, R_PERMANENT_KEY);
+  result->pk = tok->key;
+  tok->key = NULL; /* Prevent free */
+  /* Parse secret ID part. */
+  tok = find_by_keyword(tokens, R_SECRET_ID_PART);
+  tor_assert(tok->n_args == 1);
+  if (strlen(tok->args[0]) != REND_SECRET_ID_PART_LEN_BASE32 ||
+      strspn(tok->args[0], BASE32_CHARS) != REND_SECRET_ID_PART_LEN_BASE32) {
+    log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
+    goto err;
+  }
+  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
+    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
+             tok->args[0]);
+    goto err;
+  }
+  /* Parse publication time -- up-to-date check is done when storing the
+   * descriptor. */
+  tok = find_by_keyword(tokens, R_PUBLICATION_TIME);
+  tor_assert(tok->n_args == 1);
+  if (parse_iso_time_(tok->args[0], &result->timestamp,
+                      strict_time_fmt, 0) < 0) {
+    log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]);
+    goto err;
+  }
+  /* Parse protocol versions. */
+  tok = find_by_keyword(tokens, R_PROTOCOL_VERSIONS);
+  tor_assert(tok->n_args == 1);
+  versions = smartlist_new();
+  smartlist_split_string(versions, tok->args[0], ",",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  for (i = 0; i < smartlist_len(versions); i++) {
+    version = (int) tor_parse_long(smartlist_get(versions, i),
+                                   10, 0, INT_MAX, &num_ok, NULL);
+    if (!num_ok) /* It's a string; let's ignore it. */
+      continue;
+    if (version >= REND_PROTOCOL_VERSION_BITMASK_WIDTH)
+      /* Avoid undefined left-shift behaviour. */
+      continue;
+    result->protocols |= 1 << version;
+  }
+  SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp));
+  smartlist_free(versions);
+  /* Parse encrypted introduction points. Don't verify. */
+  tok = find_opt_by_keyword(tokens, R_INTRODUCTION_POINTS);
+  if (tok) {
+    if (strcmp(tok->object_type, "MESSAGE")) {
+      log_warn(LD_DIR, "Bad object type: introduction points should be of "
+               "type MESSAGE");
+      goto err;
+    }
+    *intro_points_encrypted_out = tor_memdup(tok->object_body,
+                                             tok->object_size);
+    *intro_points_encrypted_size_out = tok->object_size;
+  } else {
+    *intro_points_encrypted_out = NULL;
+    *intro_points_encrypted_size_out = 0;
+  }
+  /* Parse and verify signature. */
+  tok = find_by_keyword(tokens, R_SIGNATURE);
+  if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0,
+                            "v2 rendezvous service descriptor") < 0)
+    goto err;
+  /* Verify that descriptor ID belongs to public key and secret ID part. */
+  if (crypto_pk_get_digest(result->pk, public_key_hash) < 0) {
+    log_warn(LD_REND, "Unable to compute rend descriptor public key digest");
+    goto err;
+  }
+  rend_get_descriptor_id_bytes(test_desc_id, public_key_hash,
+                               secret_id_part);
+  if (tor_memneq(desc_id_out, test_desc_id, DIGEST_LEN)) {
+    log_warn(LD_REND, "Parsed descriptor ID does not match "
+             "computed descriptor ID.");
+    goto err;
+  }
+  goto done;
+ err:
+  rend_service_descriptor_free(result);
+  result = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area)
+    memarea_drop_all(area);
+  *parsed_out = result;
+  if (result)
+    return 0;
+  return -1;
+}
+
+/** Decrypt the encrypted introduction points in <b>ipos_encrypted</b> of
+ * length <b>ipos_encrypted_size</b> using <b>descriptor_cookie</b> and
+ * write the result to a newly allocated string that is pointed to by
+ * <b>ipos_decrypted</b> and its length to <b>ipos_decrypted_size</b>.
+ * Return 0 if decryption was successful and -1 otherwise. */
+int
+rend_decrypt_introduction_points(char **ipos_decrypted,
+                                 size_t *ipos_decrypted_size,
+                                 const char *descriptor_cookie,
+                                 const char *ipos_encrypted,
+                                 size_t ipos_encrypted_size)
+{
+  tor_assert(ipos_encrypted);
+  tor_assert(descriptor_cookie);
+  if (ipos_encrypted_size < 2) {
+    log_warn(LD_REND, "Size of encrypted introduction points is too "
+                      "small.");
+    return -1;
+  }
+  if (ipos_encrypted[0] == (int)REND_BASIC_AUTH) {
+    char iv[CIPHER_IV_LEN], client_id[REND_BASIC_AUTH_CLIENT_ID_LEN],
+         session_key[CIPHER_KEY_LEN], *dec;
+    int declen, client_blocks;
+    size_t pos = 0, len, client_entries_len;
+    crypto_digest_t *digest;
+    crypto_cipher_t *cipher;
+    client_blocks = (int) ipos_encrypted[1];
+    client_entries_len = client_blocks * REND_BASIC_AUTH_CLIENT_MULTIPLE *
+                         REND_BASIC_AUTH_CLIENT_ENTRY_LEN;
+    if (ipos_encrypted_size < 2 + client_entries_len + CIPHER_IV_LEN + 1) {
+      log_warn(LD_REND, "Size of encrypted introduction points is too "
+                        "small.");
+      return -1;
+    }
+    memcpy(iv, ipos_encrypted + 2 + client_entries_len, CIPHER_IV_LEN);
+    digest = crypto_digest_new();
+    crypto_digest_add_bytes(digest, descriptor_cookie, REND_DESC_COOKIE_LEN);
+    crypto_digest_add_bytes(digest, iv, CIPHER_IV_LEN);
+    crypto_digest_get_digest(digest, client_id,
+                             REND_BASIC_AUTH_CLIENT_ID_LEN);
+    crypto_digest_free(digest);
+    for (pos = 2; pos < 2 + client_entries_len;
+         pos += REND_BASIC_AUTH_CLIENT_ENTRY_LEN) {
+      if (tor_memeq(ipos_encrypted + pos, client_id,
+                  REND_BASIC_AUTH_CLIENT_ID_LEN)) {
+        /* Attempt to decrypt introduction points. */
+        cipher = crypto_cipher_new(descriptor_cookie);
+        if (crypto_cipher_decrypt(cipher, session_key, ipos_encrypted
+                                  + pos + REND_BASIC_AUTH_CLIENT_ID_LEN,
+                                  CIPHER_KEY_LEN) < 0) {
+          log_warn(LD_REND, "Could not decrypt session key for client.");
+          crypto_cipher_free(cipher);
+          return -1;
+        }
+        crypto_cipher_free(cipher);
+
+        len = ipos_encrypted_size - 2 - client_entries_len - CIPHER_IV_LEN;
+        dec = tor_malloc_zero(len + 1);
+        declen = crypto_cipher_decrypt_with_iv(session_key, dec, len,
+            ipos_encrypted + 2 + client_entries_len,
+            ipos_encrypted_size - 2 - client_entries_len);
+
+        if (declen < 0) {
+          log_warn(LD_REND, "Could not decrypt introduction point string.");
+          tor_free(dec);
+          return -1;
+        }
+        if (fast_memcmpstart(dec, declen, "introduction-point ")) {
+          log_warn(LD_REND, "Decrypted introduction points don't "
+                            "look like we could parse them.");
+          tor_free(dec);
+          continue;
+        }
+        *ipos_decrypted = dec;
+        *ipos_decrypted_size = declen;
+        return 0;
+      }
+    }
+    log_warn(LD_REND, "Could not decrypt introduction points. Please "
+             "check your authorization for this service!");
+    return -1;
+  } else if (ipos_encrypted[0] == (int)REND_STEALTH_AUTH) {
+    char *dec;
+    int declen;
+    if (ipos_encrypted_size < CIPHER_IV_LEN + 2) {
+      log_warn(LD_REND, "Size of encrypted introduction points is too "
+                        "small.");
+      return -1;
+    }
+    dec = tor_malloc_zero(ipos_encrypted_size - CIPHER_IV_LEN - 1 + 1);
+
+    declen = crypto_cipher_decrypt_with_iv(descriptor_cookie, dec,
+                                           ipos_encrypted_size -
+                                               CIPHER_IV_LEN - 1,
+                                           ipos_encrypted + 1,
+                                           ipos_encrypted_size - 1);
+
+    if (declen < 0) {
+      log_warn(LD_REND, "Decrypting introduction points failed!");
+      tor_free(dec);
+      return -1;
+    }
+    *ipos_decrypted = dec;
+    *ipos_decrypted_size = declen;
+    return 0;
+  } else {
+    log_warn(LD_REND, "Unknown authorization type number: %d",
+             ipos_encrypted[0]);
+    return -1;
+  }
+}
+
+/** Parse the encoded introduction points in <b>intro_points_encoded</b> of
+ * length <b>intro_points_encoded_size</b> and write the result to the
+ * descriptor in <b>parsed</b>; return the number of successfully parsed
+ * introduction points or -1 in case of a failure. */
+int
+rend_parse_introduction_points(rend_service_descriptor_t *parsed,
+                               const char *intro_points_encoded,
+                               size_t intro_points_encoded_size)
+{
+  const char *current_ipo, *end_of_intro_points;
+  smartlist_t *tokens = NULL;
+  directory_token_t *tok;
+  rend_intro_point_t *intro;
+  extend_info_t *info;
+  int result, num_ok=1;
+  memarea_t *area = NULL;
+  tor_assert(parsed);
+  /** Function may only be invoked once. */
+  tor_assert(!parsed->intro_nodes);
+  if (!intro_points_encoded || intro_points_encoded_size == 0) {
+    log_warn(LD_REND, "Empty or zero size introduction point list");
+    goto err;
+  }
+  /* Consider one intro point after the other. */
+  current_ipo = intro_points_encoded;
+  end_of_intro_points = intro_points_encoded + intro_points_encoded_size;
+  tokens = smartlist_new();
+  parsed->intro_nodes = smartlist_new();
+  area = memarea_new();
+
+  while (!fast_memcmpstart(current_ipo, end_of_intro_points-current_ipo,
+                      "introduction-point ")) {
+    /* Determine end of string. */
+    const char *eos = tor_memstr(current_ipo, end_of_intro_points-current_ipo,
+                                 "\nintroduction-point ");
+    if (!eos)
+      eos = end_of_intro_points;
+    else
+      eos = eos+1;
+    tor_assert(eos <= intro_points_encoded+intro_points_encoded_size);
+    /* Free tokens and clear token list. */
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_clear(tokens);
+    memarea_clear(area);
+    /* Tokenize string. */
+    if (tokenize_string(area, current_ipo, eos, tokens, ipo_token_table, 0)) {
+      log_warn(LD_REND, "Error tokenizing introduction point");
+      goto err;
+    }
+    /* Advance to next introduction point, if available. */
+    current_ipo = eos;
+    /* Check minimum allowed length of introduction point. */
+    if (smartlist_len(tokens) < 5) {
+      log_warn(LD_REND, "Impossibly short introduction point.");
+      goto err;
+    }
+    /* Allocate new intro point and extend info. */
+    intro = tor_malloc_zero(sizeof(rend_intro_point_t));
+    info = intro->extend_info = tor_malloc_zero(sizeof(extend_info_t));
+    /* Parse identifier. */
+    tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
+    if (base32_decode(info->identity_digest, DIGEST_LEN,
+                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
+      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
+               tok->args[0]);
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    /* Write identifier to nickname. */
+    info->nickname[0] = '$';
+    base16_encode(info->nickname + 1, sizeof(info->nickname) - 1,
+                  info->identity_digest, DIGEST_LEN);
+    /* Parse IP address. */
+    tok = find_by_keyword(tokens, R_IPO_IP_ADDRESS);
+    if (tor_addr_parse(&info->addr, tok->args[0])<0) {
+      log_warn(LD_REND, "Could not parse introduction point address.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    if (tor_addr_family(&info->addr) != AF_INET) {
+      log_warn(LD_REND, "Introduction point address was not ipv4.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+
+    /* Parse onion port. */
+    tok = find_by_keyword(tokens, R_IPO_ONION_PORT);
+    info->port = (uint16_t) tor_parse_long(tok->args[0],10,1,65535,
+                                           &num_ok,NULL);
+    if (!info->port || !num_ok) {
+      log_warn(LD_REND, "Introduction point onion port %s is invalid",
+               escaped(tok->args[0]));
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    /* Parse onion key. */
+    tok = find_by_keyword(tokens, R_IPO_ONION_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_REND,
+               "Introduction point's onion key had invalid exponent.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    info->onion_key = tok->key;
+    tok->key = NULL; /* Prevent free */
+    /* Parse service key. */
+    tok = find_by_keyword(tokens, R_IPO_SERVICE_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_REND,
+               "Introduction point key had invalid exponent.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    intro->intro_key = tok->key;
+    tok->key = NULL; /* Prevent free */
+    /* Add extend info to list of introduction points. */
+    smartlist_add(parsed->intro_nodes, intro);
+  }
+  result = smartlist_len(parsed->intro_nodes);
+  goto done;
+
+ err:
+  result = -1;
+
+ done:
+  /* Free tokens and clear token list. */
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area)
+    memarea_drop_all(area);
+
+  return result;
+}
+
+/** Parse the content of a client_key file in <b>ckstr</b> and add
+ * rend_authorized_client_t's for each parsed client to
+ * <b>parsed_clients</b>. Return the number of parsed clients as result
+ * or -1 for failure. */
+int
+rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr)
+{
+  int result = -1;
+  smartlist_t *tokens;
+  directory_token_t *tok;
+  const char *current_entry = NULL;
+  memarea_t *area = NULL;
+  char *err_msg = NULL;
+  if (!ckstr || strlen(ckstr) == 0)
+    return -1;
+  tokens = smartlist_new();
+  /* Begin parsing with first entry, skipping comments or whitespace at the
+   * beginning. */
+  area = memarea_new();
+  current_entry = eat_whitespace(ckstr);
+  while (!strcmpstart(current_entry, "client-name ")) {
+    rend_authorized_client_t *parsed_entry;
+    /* Determine end of string. */
+    const char *eos = strstr(current_entry, "\nclient-name ");
+    if (!eos)
+      eos = current_entry + strlen(current_entry);
+    else
+      eos = eos + 1;
+    /* Free tokens and clear token list. */
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_clear(tokens);
+    memarea_clear(area);
+    /* Tokenize string. */
+    if (tokenize_string(area, current_entry, eos, tokens,
+                        client_keys_token_table, 0)) {
+      log_warn(LD_REND, "Error tokenizing client keys file.");
+      goto err;
+    }
+    /* Advance to next entry, if available. */
+    current_entry = eos;
+    /* Check minimum allowed length of token list. */
+    if (smartlist_len(tokens) < 2) {
+      log_warn(LD_REND, "Impossibly short client key entry.");
+      goto err;
+    }
+    /* Parse client name. */
+    tok = find_by_keyword(tokens, C_CLIENT_NAME);
+    tor_assert(tok == smartlist_get(tokens, 0));
+    tor_assert(tok->n_args == 1);
+
+    if (!rend_valid_client_name(tok->args[0])) {
+      log_warn(LD_CONFIG, "Illegal client name: %s. (Length must be "
+               "between 1 and %d, and valid characters are "
+               "[A-Za-z0-9+-_].)", tok->args[0], REND_CLIENTNAME_MAX_LEN);
+      goto err;
+    }
+    /* Check if client name is duplicate. */
+    if (strmap_get(parsed_clients, tok->args[0])) {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains a "
+               "duplicate client name: '%s'. Ignoring.", tok->args[0]);
+      goto err;
+    }
+    parsed_entry = tor_malloc_zero(sizeof(rend_authorized_client_t));
+    parsed_entry->client_name = tor_strdup(tok->args[0]);
+    strmap_set(parsed_clients, parsed_entry->client_name, parsed_entry);
+    /* Parse client key. */
+    tok = find_opt_by_keyword(tokens, C_CLIENT_KEY);
+    if (tok) {
+      parsed_entry->client_key = tok->key;
+      tok->key = NULL; /* Prevent free */
+    }
+
+    /* Parse descriptor cookie. */
+    tok = find_by_keyword(tokens, C_DESCRIPTOR_COOKIE);
+    tor_assert(tok->n_args == 1);
+    if (rend_auth_decode_cookie(tok->args[0], parsed_entry->descriptor_cookie,
+                                NULL, &err_msg) < 0) {
+      tor_assert(err_msg);
+      log_warn(LD_REND, "%s", err_msg);
+      tor_free(err_msg);
+      goto err;
+    }
+  }
+  result = strmap_size(parsed_clients);
+  goto done;
+ err:
+  result = -1;
+ done:
+  /* Free tokens and clear token list. */
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area)
+    memarea_drop_all(area);
+  return result;
+}
+
+/** Called on startup; right now we just handle scanning the unparseable
+ * descriptor dumps, but hang anything else we might need to do in the
+ * future here as well.
+ */
+void
+routerparse_init(void)
+{
+  /*
+   * Check both if the sandbox is active and whether it's configured; no
+   * point in loading all that if we won't be able to use it after the
+   * sandbox becomes active.
+   */
+  if (!(sandbox_is_active() || get_options()->Sandbox)) {
+    dump_desc_init();
+  }
+}
+
+/** Clean up all data structures used by routerparse.c at exit */
+void
+routerparse_free_all(void)
+{
+  dump_desc_fifo_cleanup();
+}
diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h
new file mode 100644
index 0000000000..87c2a75aa5
--- /dev/null
+++ b/src/feature/nodelist/routerparse.h
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerparse.h
+ * \brief Header file for routerparse.c.
+ **/
+
+#ifndef TOR_ROUTERPARSE_H
+#define TOR_ROUTERPARSE_H
+
+/** Possible statuses of a version of Tor, given opinions from the directory
+ * servers. */
+typedef enum version_status_t {
+  VS_RECOMMENDED=0, /**< This version is listed as recommended. */
+  VS_OLD=1, /**< This version is older than any recommended version. */
+  VS_NEW=2, /**< This version is newer than any recommended version. */
+  VS_NEW_IN_SERIES=3, /**< This version is newer than any recommended version
+                       * in its series, but later recommended versions exist.
+                       */
+  VS_UNRECOMMENDED=4, /**< This version is not recommended (general case). */
+  VS_EMPTY=5, /**< The version list was empty; no agreed-on versions. */
+  VS_UNKNOWN, /**< We have no idea. */
+} version_status_t;
+
+enum networkstatus_type_t;
+
+int router_get_router_hash(const char *s, size_t s_len, char *digest);
+int router_get_dir_hash(const char *s, char *digest);
+int router_get_networkstatus_v3_hashes(const char *s,
+                                       common_digests_t *digests);
+int router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                                  const char **start_out,
+                                                  const char **end_out);
+int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                               const char *s);
+int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest);
+#define DIROBJ_MAX_SIG_LEN 256
+char *router_get_dirobj_signature(const char *digest,
+                                  size_t digest_len,
+                                  const crypto_pk_t *private_key);
+int router_append_dirobj_signature(char *buf, size_t buf_len,
+                                   const char *digest,
+                                   size_t digest_len,
+                                   crypto_pk_t *private_key);
+int router_parse_list_from_string(const char **s, const char *eos,
+                                  smartlist_t *dest,
+                                  saved_location_t saved_location,
+                                  int is_extrainfo,
+                                  int allow_annotations,
+                                  const char *prepend_annotations,
+                                  smartlist_t *invalid_digests_out);
+
+routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
+                                             int cache_copy,
+                                             int allow_annotations,
+                                             const char *prepend_annotations,
+                                             int *can_dl_again_out);
+struct digest_ri_map_t;
+extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end,
+                             int cache_copy, struct digest_ri_map_t *routermap,
+                             int *can_dl_again_out);
+MOCK_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
+         (const char *s, int assume_action, int *malformed_list));
+version_status_t tor_version_is_obsolete(const char *myversion,
+                                         const char *versionlist);
+int tor_version_parse_platform(const char *platform,
+                               tor_version_t *version_out,
+                               int strict);
+int tor_version_as_new_as(const char *platform, const char *cutoff);
+int tor_version_parse(const char *s, tor_version_t *out);
+int tor_version_compare(tor_version_t *a, tor_version_t *b);
+int tor_version_same_series(tor_version_t *a, tor_version_t *b);
+void sort_version_list(smartlist_t *lst, int remove_duplicates);
+void assert_addr_policy_ok(smartlist_t *t);
+void dump_distinct_digest_count(int severity);
+
+int compare_vote_routerstatus_entries(const void **_a, const void **_b);
+int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
+networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
+                                           const char **eos_out,
+                                           enum networkstatus_type_t ns_type);
+ns_detached_signatures_t *networkstatus_parse_detached_signatures(
+                                          const char *s, const char *eos);
+
+smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
+                                          int allow_annotations,
+                                          saved_location_t where,
+                                          smartlist_t *invalid_digests_out);
+
+authority_cert_t *authority_cert_parse_from_string(const char *s,
+                                                   const char **end_of_string);
+int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
+                                     char *desc_id_out,
+                                     char **intro_points_encrypted_out,
+                                     size_t *intro_points_encrypted_size_out,
+                                     size_t *encoded_size_out,
+                                     const char **next_out, const char *desc,
+                                     int as_hsdir);
+int rend_decrypt_introduction_points(char **ipos_decrypted,
+                                     size_t *ipos_decrypted_size,
+                                     const char *descriptor_cookie,
+                                     const char *ipos_encrypted,
+                                     size_t ipos_encrypted_size);
+int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
+                                   const char *intro_points_encoded,
+                                   size_t intro_points_encoded_size);
+int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
+
+void routerparse_init(void);
+void routerparse_free_all(void);
+
+#ifdef ROUTERPARSE_PRIVATE
+/*
+ * One entry in the list of dumped descriptors; filename dumped to, length,
+ * SHA-256 and timestamp.
+ */
+
+typedef struct {
+  char *filename;
+  size_t len;
+  uint8_t digest_sha256[DIGEST256_LEN];
+  time_t when;
+} dumped_desc_t;
+
+EXTERN(uint64_t, len_descs_dumped)
+EXTERN(smartlist_t *, descs_dumped)
+STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
+                                            networkstatus_t *vote,
+                                            vote_routerstatus_t *vote_rs,
+                                            routerstatus_t *rs);
+MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
+    (const char *dirname, const char *f));
+STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
+STATIC void dump_desc_fifo_cleanup(void);
+struct memarea_t;
+STATIC routerstatus_t *routerstatus_parse_entry_from_string(
+                                     struct memarea_t *area,
+                                     const char **s, smartlist_t *tokens,
+                                     networkstatus_t *vote,
+                                     vote_routerstatus_t *vote_rs,
+                                     int consensus_method,
+                                     consensus_flavor_t flav);
+MOCK_DECL(STATIC void,dump_desc,(const char *desc, const char *type));
+MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg));
+MOCK_DECL(STATIC int, signed_digest_equals,
+          (const uint8_t *d1, const uint8_t *d2, size_t len));
+
+STATIC void summarize_protover_flags(protover_summary_flags_t *out,
+                                     const char *protocols,
+                                     const char *version);
+#endif /* defined(ROUTERPARSE_PRIVATE) */
+
+#define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1"
+
+#endif /* !defined(TOR_ROUTERPARSE_H) */
diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c
new file mode 100644
index 0000000000..285ef9d821
--- /dev/null
+++ b/src/feature/nodelist/routerset.c
@@ -0,0 +1,463 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+n * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerset.c
+ *
+ * \brief Functions and structures to handle set-type selection of routers
+ *  by name, ID, address, etc.
+ *
+ * This module implements the routerset_t data structure, whose purpose
+ * is to specify a set of relays based on a list of their identities or
+ * properties.  Routersets can restrict relays by IP address mask,
+ * identity fingerprint, country codes, and nicknames (deprecated).
+ *
+ * Routersets are typically used for user-specified restrictions, and
+ * are created by invoking routerset_new and routerset_parse from
+ * config.c and confparse.c.  To use a routerset, invoke one of
+ * routerset_contains_...() functions , or use
+ * routerstatus_get_all_nodes() / routerstatus_subtract_nodes() to
+ * manipulate a smartlist of node_t pointers.
+ *
+ * Country-code restrictions are implemented in geoip.c.
+ */
+
+#define ROUTERSET_PRIVATE
+
+#include "or/or.h"
+#include "or/bridges.h"
+#include "or/geoip.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/router.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+
+#include "or/addr_policy_st.h"
+#include "or/extend_info_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerstatus_st.h"
+
+/** Return a new empty routerset. */
+routerset_t *
+routerset_new(void)
+{
+  routerset_t *result = tor_malloc_zero(sizeof(routerset_t));
+  result->list = smartlist_new();
+  result->names = strmap_new();
+  result->digests = digestmap_new();
+  result->policies = smartlist_new();
+  result->country_names = smartlist_new();
+  return result;
+}
+
+/** If <b>c</b> is a country code in the form {cc}, return a newly allocated
+ * string holding the "cc" part.  Else, return NULL. */
+STATIC char *
+routerset_get_countryname(const char *c)
+{
+  char *country;
+
+  if (strlen(c) < 4 || c[0] !='{' || c[3] !='}')
+    return NULL;
+
+  country = tor_strndup(c+1, 2);
+  tor_strlower(country);
+  return country;
+}
+
+/** Update the routerset's <b>countries</b> bitarray_t. Called whenever
+ * the GeoIP IPv4 database is reloaded.
+ */
+void
+routerset_refresh_countries(routerset_t *target)
+{
+  int cc;
+  bitarray_free(target->countries);
+
+  if (!geoip_is_loaded(AF_INET)) {
+    target->countries = NULL;
+    target->n_countries = 0;
+    return;
+  }
+  target->n_countries = geoip_get_n_countries();
+  target->countries = bitarray_init_zero(target->n_countries);
+  SMARTLIST_FOREACH_BEGIN(target->country_names, const char *, country) {
+    cc = geoip_get_country(country);
+    if (cc >= 0) {
+      tor_assert(cc < target->n_countries);
+      bitarray_set(target->countries, cc);
+    } else {
+      log_warn(LD_CONFIG, "Country code '%s' is not recognized.",
+          country);
+    }
+  } SMARTLIST_FOREACH_END(country);
+}
+
+/** Parse the string <b>s</b> to create a set of routerset entries, and add
+ * them to <b>target</b>.  In log messages, refer to the string as
+ * <b>description</b>.  Return 0 on success, -1 on failure.
+ *
+ * Three kinds of elements are allowed in routersets: nicknames, IP address
+ * patterns, and fingerprints.  They may be surrounded by optional space, and
+ * must be separated by commas.
+ */
+int
+routerset_parse(routerset_t *target, const char *s, const char *description)
+{
+  int r = 0;
+  int added_countries = 0;
+  char *countryname;
+  smartlist_t *list = smartlist_new();
+  int malformed_list;
+  smartlist_split_string(list, s, ",",
+                         SPLIT_SKIP_SPACE | SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(list, char *, nick) {
+      addr_policy_t *p;
+      /* if it doesn't pass our validation, assume it's malformed */
+      malformed_list = 1;
+      if (is_legal_hexdigest(nick)) {
+        char d[DIGEST_LEN];
+        if (*nick == '$')
+          ++nick;
+        log_debug(LD_CONFIG, "Adding identity %s to %s", nick, description);
+        base16_decode(d, sizeof(d), nick, HEX_DIGEST_LEN);
+        digestmap_set(target->digests, d, (void*)1);
+      } else if (is_legal_nickname(nick)) {
+        log_debug(LD_CONFIG, "Adding nickname %s to %s", nick, description);
+        strmap_set_lc(target->names, nick, (void*)1);
+      } else if ((countryname = routerset_get_countryname(nick)) != NULL) {
+        log_debug(LD_CONFIG, "Adding country %s to %s", nick,
+                  description);
+        smartlist_add(target->country_names, countryname);
+        added_countries = 1;
+      } else if ((strchr(nick,'.') || strchr(nick, ':') ||  strchr(nick, '*'))
+                 && (p = router_parse_addr_policy_item_from_string(
+                                     nick, ADDR_POLICY_REJECT,
+                                     &malformed_list))) {
+        /* IPv4 addresses contain '.', IPv6 addresses contain ':',
+         * and wildcard addresses contain '*'. */
+        log_debug(LD_CONFIG, "Adding address %s to %s", nick, description);
+        smartlist_add(target->policies, p);
+      } else if (malformed_list) {
+        log_warn(LD_CONFIG, "Entry '%s' in %s is malformed. Discarding entire"
+                 " list.", nick, description);
+        r = -1;
+        tor_free(nick);
+        SMARTLIST_DEL_CURRENT(list, nick);
+      } else {
+        log_notice(LD_CONFIG, "Entry '%s' in %s is ignored. Using the"
+                   " remainder of the list.", nick, description);
+        tor_free(nick);
+        SMARTLIST_DEL_CURRENT(list, nick);
+      }
+  } SMARTLIST_FOREACH_END(nick);
+  policy_expand_unspec(&target->policies);
+  smartlist_add_all(target->list, list);
+  smartlist_free(list);
+  if (added_countries)
+    routerset_refresh_countries(target);
+  return r;
+}
+
+/** Add all members of the set <b>source</b> to <b>target</b>. */
+void
+routerset_union(routerset_t *target, const routerset_t *source)
+{
+  char *s;
+  tor_assert(target);
+  if (!source || !source->list)
+    return;
+  s = routerset_to_string(source);
+  routerset_parse(target, s, "other routerset");
+  tor_free(s);
+}
+
+/** Return true iff <b>set</b> lists only nicknames and digests, and includes
+ * no IP ranges or countries. */
+int
+routerset_is_list(const routerset_t *set)
+{
+  return smartlist_len(set->country_names) == 0 &&
+    smartlist_len(set->policies) == 0;
+}
+
+/** Return true iff we need a GeoIP IP-to-country database to make sense of
+ * <b>set</b>. */
+int
+routerset_needs_geoip(const routerset_t *set)
+{
+  return set && smartlist_len(set->country_names);
+}
+
+/** Return true iff there are no entries in <b>set</b>. */
+int
+routerset_is_empty(const routerset_t *set)
+{
+  return !set || smartlist_len(set->list) == 0;
+}
+
+/** Return the number of entries in <b>set</b>. This does NOT return a
+ * negative value. */
+int
+routerset_len(const routerset_t *set)
+{
+  if (!set) {
+    return 0;
+  }
+  return smartlist_len(set->list);
+}
+
+/** Helper.  Return true iff <b>set</b> contains a router based on the other
+ * provided fields.  Return higher values for more specific subentries: a
+ * single router is more specific than an address range of routers, which is
+ * more specific in turn than a country code.
+ *
+ * (If country is -1, then we take the country
+ * from addr.) */
+STATIC int
+routerset_contains(const routerset_t *set, const tor_addr_t *addr,
+                   uint16_t orport,
+                   const char *nickname, const char *id_digest,
+                   country_t country)
+{
+  if (!set || !set->list)
+    return 0;
+  if (nickname && strmap_get_lc(set->names, nickname))
+    return 4;
+  if (id_digest && digestmap_get(set->digests, id_digest))
+    return 4;
+  if (addr && compare_tor_addr_to_addr_policy(addr, orport, set->policies)
+      == ADDR_POLICY_REJECTED)
+    return 3;
+  if (set->countries) {
+    if (country < 0 && addr)
+      country = geoip_get_country_by_addr(addr);
+
+    if (country >= 0 && country < set->n_countries &&
+        bitarray_is_set(set->countries, country))
+      return 2;
+  }
+  return 0;
+}
+
+/** If *<b>setp</b> includes at least one country code, or if
+ * <b>only_some_cc_set</b> is 0, add the ?? and A1 country codes to
+ * *<b>setp</b>, creating it as needed.  Return true iff *<b>setp</b> changed.
+ */
+int
+routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set)
+{
+  routerset_t *set;
+  int add_unknown, add_a1;
+  if (only_if_some_cc_set) {
+    if (!*setp || smartlist_len((*setp)->country_names) == 0)
+      return 0;
+  }
+  if (!*setp)
+    *setp = routerset_new();
+
+  set = *setp;
+
+  add_unknown = ! smartlist_contains_string_case(set->country_names, "??") &&
+    geoip_get_country("??") >= 0;
+  add_a1 = ! smartlist_contains_string_case(set->country_names, "a1") &&
+    geoip_get_country("A1") >= 0;
+
+  if (add_unknown) {
+    smartlist_add_strdup(set->country_names, "??");
+    smartlist_add_strdup(set->list, "{??}");
+  }
+  if (add_a1) {
+    smartlist_add_strdup(set->country_names, "a1");
+    smartlist_add_strdup(set->list, "{a1}");
+  }
+
+  if (add_unknown || add_a1) {
+    routerset_refresh_countries(set);
+    return 1;
+  }
+  return 0;
+}
+
+/** Return true iff we can tell that <b>ei</b> is a member of <b>set</b>. */
+int
+routerset_contains_extendinfo(const routerset_t *set, const extend_info_t *ei)
+{
+  return routerset_contains(set,
+                            &ei->addr,
+                            ei->port,
+                            ei->nickname,
+                            ei->identity_digest,
+                            -1 /*country*/);
+}
+
+/** Return true iff <b>ri</b> is in <b>set</b>.  If country is <b>-1</b>, we
+ * look up the country. */
+int
+routerset_contains_router(const routerset_t *set, const routerinfo_t *ri,
+                          country_t country)
+{
+  tor_addr_t addr;
+  tor_addr_from_ipv4h(&addr, ri->addr);
+  return routerset_contains(set,
+                            &addr,
+                            ri->or_port,
+                            ri->nickname,
+                            ri->cache_info.identity_digest,
+                            country);
+}
+
+/** Return true iff <b>rs</b> is in <b>set</b>.  If country is <b>-1</b>, we
+ * look up the country. */
+int
+routerset_contains_routerstatus(const routerset_t *set,
+                                const routerstatus_t *rs,
+                                country_t country)
+{
+  tor_addr_t addr;
+  tor_addr_from_ipv4h(&addr, rs->addr);
+  return routerset_contains(set,
+                            &addr,
+                            rs->or_port,
+                            rs->nickname,
+                            rs->identity_digest,
+                            country);
+}
+
+/** Return true iff <b>node</b> is in <b>set</b>. */
+int
+routerset_contains_node(const routerset_t *set, const node_t *node)
+{
+  if (node->rs)
+    return routerset_contains_routerstatus(set, node->rs, node->country);
+  else if (node->ri)
+    return routerset_contains_router(set, node->ri, node->country);
+  else
+    return 0;
+}
+
+/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
+int
+routerset_contains_bridge(const routerset_t *set, const bridge_info_t *bridge)
+{
+  const char *id = (const char*)bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  tor_assert(addrport);
+  return routerset_contains(set, &addrport->addr, addrport->port,
+                            NULL, id, -1);
+}
+
+/** Add every known node_t that is a member of <b>routerset</b> to
+ * <b>out</b>, but never add any that are part of <b>excludeset</b>.
+ * If <b>running_only</b>, only add the running ones. */
+void
+routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
+                        const routerset_t *excludeset, int running_only)
+{
+  tor_assert(out);
+  if (!routerset || !routerset->list)
+    return;
+
+  if (routerset_is_list(routerset)) {
+    /* No routers are specified by type; all are given by name or digest.
+     * we can do a lookup in O(len(routerset)). */
+    SMARTLIST_FOREACH(routerset->list, const char *, name, {
+        const node_t *node = node_get_by_nickname(name, 0);
+        if (node) {
+          if (!running_only || node->is_running)
+            if (!routerset_contains_node(excludeset, node))
+              smartlist_add(out, (void*)node);
+        }
+    });
+  } else {
+    /* We need to iterate over the routerlist to get all the ones of the
+     * right kind. */
+    smartlist_t *nodes = nodelist_get_list();
+    SMARTLIST_FOREACH(nodes, const node_t *, node, {
+        if (running_only && !node->is_running)
+          continue;
+        if (routerset_contains_node(routerset, node) &&
+            !routerset_contains_node(excludeset, node))
+          smartlist_add(out, (void*)node);
+    });
+  }
+}
+
+/** Remove every node_t from <b>lst</b> that is in <b>routerset</b>. */
+void
+routerset_subtract_nodes(smartlist_t *lst, const routerset_t *routerset)
+{
+  tor_assert(lst);
+  if (!routerset)
+    return;
+  SMARTLIST_FOREACH(lst, const node_t *, node, {
+      if (routerset_contains_node(routerset, node)) {
+        //log_debug(LD_DIR, "Subtracting %s",r->nickname);
+        SMARTLIST_DEL_CURRENT(lst, node);
+      }
+    });
+}
+
+/** Return a new string that when parsed by routerset_parse_string() will
+ * yield <b>set</b>. */
+char *
+routerset_to_string(const routerset_t *set)
+{
+  if (!set || !set->list)
+    return tor_strdup("");
+  return smartlist_join_strings(set->list, ",", 0, NULL);
+}
+
+/** Helper: return true iff old and new are both NULL, or both non-NULL
+ * equal routersets. */
+int
+routerset_equal(const routerset_t *old, const routerset_t *new)
+{
+  if (routerset_is_empty(old) && routerset_is_empty(new)) {
+    /* Two empty sets are equal */
+    return 1;
+  } else if (routerset_is_empty(old) || routerset_is_empty(new)) {
+    /* An empty set is equal to nothing else. */
+    return 0;
+  }
+  tor_assert(old != NULL);
+  tor_assert(new != NULL);
+
+  if (smartlist_len(old->list) != smartlist_len(new->list))
+    return 0;
+
+  SMARTLIST_FOREACH(old->list, const char *, cp1, {
+    const char *cp2 = smartlist_get(new->list, cp1_sl_idx);
+    if (strcmp(cp1, cp2))
+      return 0;
+  });
+
+  return 1;
+}
+
+/** Free all storage held in <b>routerset</b>. */
+void
+routerset_free_(routerset_t *routerset)
+{
+  if (!routerset)
+    return;
+
+  SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp));
+  smartlist_free(routerset->list);
+  SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p,
+                    addr_policy_free(p));
+  smartlist_free(routerset->policies);
+  SMARTLIST_FOREACH(routerset->country_names, char *, cp, tor_free(cp));
+  smartlist_free(routerset->country_names);
+
+  strmap_free(routerset->names, NULL);
+  digestmap_free(routerset->digests, NULL);
+  bitarray_free(routerset->countries);
+  tor_free(routerset);
+}
diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h
new file mode 100644
index 0000000000..8a13ca042a
--- /dev/null
+++ b/src/feature/nodelist/routerset.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerlist.h
+ * \brief Header file for routerset.c
+ **/
+
+#ifndef TOR_ROUTERSET_H
+#define TOR_ROUTERSET_H
+
+routerset_t *routerset_new(void);
+void routerset_refresh_countries(routerset_t *rs);
+int routerset_parse(routerset_t *target, const char *s,
+                    const char *description);
+void routerset_union(routerset_t *target, const routerset_t *source);
+int routerset_is_list(const routerset_t *set);
+int routerset_needs_geoip(const routerset_t *set);
+int routerset_is_empty(const routerset_t *set);
+int routerset_contains_router(const routerset_t *set, const routerinfo_t *ri,
+                              country_t country);
+int routerset_contains_routerstatus(const routerset_t *set,
+                                    const routerstatus_t *rs,
+                                    country_t country);
+int routerset_contains_extendinfo(const routerset_t *set,
+                                  const extend_info_t *ei);
+struct bridge_info_t;
+int routerset_contains_bridge(const routerset_t *set,
+                              const struct bridge_info_t *bridge);
+int routerset_contains_node(const routerset_t *set, const node_t *node);
+
+void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
+                             const routerset_t *excludeset,
+                             int running_only);
+int routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set);
+void routerset_subtract_nodes(smartlist_t *out,
+                                const routerset_t *routerset);
+
+char *routerset_to_string(const routerset_t *routerset);
+int routerset_equal(const routerset_t *old, const routerset_t *new);
+void routerset_free_(routerset_t *routerset);
+#define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
+int routerset_len(const routerset_t *set);
+
+#ifdef ROUTERSET_PRIVATE
+#include "lib/container/bitarray.h"
+
+STATIC char * routerset_get_countryname(const char *c);
+STATIC int routerset_contains(const routerset_t *set, const tor_addr_t *addr,
+                   uint16_t orport,
+                   const char *nickname, const char *id_digest,
+                   country_t country);
+
+/** A routerset specifies constraints on a set of possible routerinfos, based
+ * on their names, identities, or addresses.  It is optimized for determining
+ * whether a router is a member or not, in O(1+P) time, where P is the number
+ * of address policy constraints. */
+struct routerset_t {
+  /** A list of strings for the elements of the policy.  Each string is either
+   * a nickname, a hexadecimal identity fingerprint, or an address policy.  A
+   * router belongs to the set if its nickname OR its identity OR its address
+   * matches an entry here. */
+  smartlist_t *list;
+  /** A map from lowercase nicknames of routers in the set to (void*)1 */
+  strmap_t *names;
+  /** A map from identity digests routers in the set to (void*)1 */
+  digestmap_t *digests;
+  /** An address policy for routers in the set.  For implementation reasons,
+   * a router belongs to the set if it is _rejected_ by this policy. */
+  smartlist_t *policies;
+
+  /** A human-readable description of what this routerset is for.  Used in
+   * log messages. */
+  char *description;
+
+  /** A list of the country codes in this set. */
+  smartlist_t *country_names;
+  /** Total number of countries we knew about when we built <b>countries</b>.*/
+  int n_countries;
+  /** Bit array mapping the return value of geoip_get_country() to 1 iff the
+   * country is a member of this routerset.  Note that we MUST call
+   * routerset_refresh_countries() whenever the geoip country list is
+   * reloaded. */
+  bitarray_t *countries;
+};
+#endif /* defined(ROUTERSET_PRIVATE) */
+#endif /* !defined(TOR_ROUTERSET_H) */
diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h
new file mode 100644
index 0000000000..3de4a40ae4
--- /dev/null
+++ b/src/feature/nodelist/routerstatus_st.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERSTATUS_ST_H
+#define ROUTERSTATUS_ST_H
+
+#include "or/download_status_st.h"
+
+/** Contents of a single router entry in a network status object.
+ */
+struct routerstatus_t {
+  time_t published_on; /**< When was this router published? */
+  char nickname[MAX_NICKNAME_LEN+1]; /**< The nickname this router says it
+                                      * has. */
+  char identity_digest[DIGEST_LEN]; /**< Digest of the router's identity
+                                     * key. */
+  /** Digest of the router's most recent descriptor or microdescriptor.
+   * If it's a descriptor, we only use the first DIGEST_LEN bytes. */
+  char descriptor_digest[DIGEST256_LEN];
+  uint32_t addr; /**< IPv4 address for this router, in host order. */
+  uint16_t or_port; /**< IPv4 OR port for this router. */
+  uint16_t dir_port; /**< Directory port for this router. */
+  tor_addr_t ipv6_addr; /**< IPv6 address for this router. */
+  uint16_t ipv6_orport; /**< IPv6 OR port for this router. */
+  unsigned int is_authority:1; /**< True iff this router is an authority. */
+  unsigned int is_exit:1; /**< True iff this router is a good exit. */
+  unsigned int is_stable:1; /**< True iff this router stays up a long time. */
+  unsigned int is_fast:1; /**< True iff this router has good bandwidth. */
+  /** True iff this router is called 'running' in the consensus. We give it
+   * this funny name so that we don't accidentally use this bit as a view of
+   * whether we think the router is *currently* running.  If that's what you
+   * want to know, look at is_running in node_t. */
+  unsigned int is_flagged_running:1;
+  unsigned int is_named:1; /**< True iff "nickname" belongs to this router. */
+  unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another
+                              * router. */
+  unsigned int is_valid:1; /**< True iff this router isn't invalid. */
+  unsigned int is_possible_guard:1; /**< True iff this router would be a good
+                                     * choice as an entry guard. */
+  unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for
+                               * an exit node. */
+  unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden
+                             * service directory. */
+  unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
+                             * or it claims to accept tunnelled dir requests.
+                             */
+
+  unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
+  unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
+  unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with
+                                    * the Unmeasured flag set. */
+
+  /** Flags to summarize the protocol versions for this routerstatus_t. */
+  protover_summary_flags_t pv;
+
+  uint32_t bandwidth_kb; /**< Bandwidth (capacity) of the router as reported in
+                       * the vote/consensus, in kilobytes/sec. */
+
+  /** The consensus has guardfraction information for this router. */
+  unsigned int has_guardfraction:1;
+  /** The guardfraction value of this router. */
+  uint32_t guardfraction_percentage;
+
+  char *exitsummary; /**< exit policy summary -
+                      * XXX weasel: this probably should not stay a string. */
+
+  /* ---- The fields below aren't derived from the networkstatus; they
+   * hold local information only. */
+
+  time_t last_dir_503_at; /**< When did this router last tell us that it
+                           * was too busy to serve directory info? */
+  download_status_t dl_status;
+
+};
+
+#endif
+
diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h
new file mode 100644
index 0000000000..90cd4a2703
--- /dev/null
+++ b/src/feature/nodelist/signed_descriptor_st.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef SIGNED_DESCRIPTOR_ST_H
+#define SIGNED_DESCRIPTOR_ST_H
+
+#include "or/download_status_st.h"
+
+/** Information need to cache an onion router's descriptor. */
+struct signed_descriptor_t {
+  /** Pointer to the raw server descriptor, preceded by annotations.  Not
+   * necessarily NUL-terminated.  If saved_location is SAVED_IN_CACHE, this
+   * pointer is null. */
+  char *signed_descriptor_body;
+  /** Length of the annotations preceding the server descriptor. */
+  size_t annotations_len;
+  /** Length of the server descriptor. */
+  size_t signed_descriptor_len;
+  /** Digest of the server descriptor, computed as specified in
+   * dir-spec.txt. */
+  char signed_descriptor_digest[DIGEST_LEN];
+  /** Identity digest of the router. */
+  char identity_digest[DIGEST_LEN];
+  /** Declared publication time of the descriptor. */
+  time_t published_on;
+  /** For routerdescs only: digest of the corresponding extrainfo. */
+  char extra_info_digest[DIGEST_LEN];
+  /** For routerdescs only: A SHA256-digest of the extrainfo (if any) */
+  char extra_info_digest256[DIGEST256_LEN];
+  /** Certificate for ed25519 signing key. */
+  struct tor_cert_st *signing_key_cert;
+  /** For routerdescs only: Status of downloading the corresponding
+   * extrainfo. */
+  download_status_t ei_dl_status;
+  /** Where is the descriptor saved? */
+  saved_location_t saved_location;
+  /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
+   * this descriptor in the corresponding file. */
+  off_t saved_offset;
+  /** What position is this descriptor within routerlist->routers or
+   * routerlist->old_routers? -1 for none. */
+  int routerlist_index;
+  /** The valid-until time of the most recent consensus that listed this
+   * descriptor.  0 for "never listed in a consensus, so far as we know." */
+  time_t last_listed_as_valid_until;
+  /* If true, we do not ever try to save this object in the cache. */
+  unsigned int do_not_cache : 1;
+  /* If true, this item is meant to represent an extrainfo. */
+  unsigned int is_extrainfo : 1;
+  /* If true, we got an extrainfo for this item, and the digest was right,
+   * but it was incompatible. */
+  unsigned int extrainfo_is_bogus : 1;
+  /* If true, we are willing to transmit this item unencrypted. */
+  unsigned int send_unencrypted : 1;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c
new file mode 100644
index 0000000000..39c6605c65
--- /dev/null
+++ b/src/feature/nodelist/torcert.c
@@ -0,0 +1,725 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torcert.c
+ *
+ * \brief Implementation for ed25519-signed certificates as used in the Tor
+ * protocol.
+ *
+ * This certificate format is designed to be simple and compact; it's
+ * documented in tor-spec.txt in the torspec.git repository.  All of the
+ * certificates in this format are signed with an Ed25519 key; the
+ * contents themselves may be another Ed25519 key, a digest of a
+ * RSA key, or some other material.
+ *
+ * In this module there is also support for a crooss-certification of
+ * Ed25519 identities using (older) RSA1024 identities.
+ *
+ * Tor uses other types of certificate too, beyond those described in this
+ * module. Notably, our use of TLS requires us to touch X.509 certificates,
+ * even though sensible people would stay away from those. Our X.509
+ * certificates are represented with tor_x509_cert_t, and implemented in
+ * tortls.c.  We also have a separate certificate type that authorities
+ * use to authenticate their RSA signing keys with their RSA identity keys:
+ * that one is authority_cert_t, and it's mostly handled in routerlist.c.
+ */
+
+#include "or/or.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/torcert.h"
+#include "trunnel/ed25519_cert.h"
+#include "lib/log/torlog.h"
+#include "trunnel/link_handshake.h"
+#include "lib/tls/tortls.h"
+
+#include "or/or_handshake_certs_st.h"
+
+/** Helper for tor_cert_create(): signs any 32 bytes, not just an ed25519
+ * key.
+ */
+static tor_cert_t *
+tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
+                      uint8_t cert_type,
+                      uint8_t signed_key_type,
+                      const uint8_t signed_key_info[32],
+                      time_t now, time_t lifetime,
+                      uint32_t flags)
+{
+  tor_cert_t *torcert = NULL;
+
+  ed25519_cert_t *cert = ed25519_cert_new();
+  cert->cert_type = cert_type;
+  cert->exp_field = (uint32_t) CEIL_DIV(now + lifetime, 3600);
+  cert->cert_key_type = signed_key_type;
+  memcpy(cert->certified_key, signed_key_info, 32);
+
+  if (flags & CERT_FLAG_INCLUDE_SIGNING_KEY) {
+    ed25519_cert_extension_t *ext = ed25519_cert_extension_new();
+    ext->ext_type = CERTEXT_SIGNED_WITH_KEY;
+    memcpy(ext->un_signing_key, signing_key->pubkey.pubkey, 32);
+    ed25519_cert_add_ext(cert, ext);
+    ++cert->n_extensions;
+  }
+
+  const ssize_t alloc_len = ed25519_cert_encoded_len(cert);
+  tor_assert(alloc_len > 0);
+  uint8_t *encoded = tor_malloc(alloc_len);
+  const ssize_t real_len = ed25519_cert_encode(encoded, alloc_len, cert);
+  if (real_len < 0)
+    goto err;
+  tor_assert(real_len == alloc_len);
+  tor_assert(real_len > ED25519_SIG_LEN);
+  uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
+  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
+
+  ed25519_signature_t signature;
+  if (ed25519_sign(&signature, encoded,
+                   real_len-ED25519_SIG_LEN, signing_key)<0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Can't sign certificate");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+  memcpy(sig, signature.sig, ED25519_SIG_LEN);
+
+  torcert = tor_cert_parse(encoded, real_len);
+  if (! torcert) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Generated a certificate we cannot parse");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (tor_cert_checksig(torcert, &signing_key->pubkey, now) < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Generated a certificate whose signature we can't "
+             "check: %s", tor_cert_describe_signature_status(torcert));
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  tor_free(encoded);
+
+  goto done;
+
+ /* LCOV_EXCL_START */
+ err:
+  tor_cert_free(torcert);
+  torcert = NULL;
+ /* LCOV_EXCL_STOP */
+
+ done:
+  ed25519_cert_free(cert);
+  tor_free(encoded);
+  return torcert;
+}
+
+/**
+ * Create and return a new new certificate of type <b>cert_type</b> to
+ * authenticate <b>signed_key</b> using the key <b>signing_key</b>.  The
+ * certificate should remain valid for at least <b>lifetime</b> seconds after
+ * <b>now</b>.
+ *
+ * If CERT_FLAG_INCLUDE_SIGNING_KEY is set in <b>flags</b>, embed
+ * the public part of <b>signing_key</b> in the certificate.
+ */
+tor_cert_t *
+tor_cert_create(const ed25519_keypair_t *signing_key,
+                uint8_t cert_type,
+                const ed25519_public_key_t *signed_key,
+                time_t now, time_t lifetime,
+                uint32_t flags)
+{
+  return tor_cert_sign_impl(signing_key, cert_type,
+                            SIGNED_KEY_TYPE_ED25519, signed_key->pubkey,
+                            now, lifetime, flags);
+}
+
+/** Release all storage held for <b>cert</b>. */
+void
+tor_cert_free_(tor_cert_t *cert)
+{
+  if (! cert)
+    return;
+
+  if (cert->encoded)
+    memwipe(cert->encoded, 0, cert->encoded_len);
+  tor_free(cert->encoded);
+
+  memwipe(cert, 0, sizeof(tor_cert_t));
+  tor_free(cert);
+}
+
+/** Parse a certificate encoded with <b>len</b> bytes in <b>encoded</b>. */
+tor_cert_t *
+tor_cert_parse(const uint8_t *encoded, const size_t len)
+{
+  tor_cert_t *cert = NULL;
+  ed25519_cert_t *parsed = NULL;
+  ssize_t got_len = ed25519_cert_parse(&parsed, encoded, len);
+  if (got_len < 0 || (size_t) got_len != len)
+    goto err;
+
+  cert = tor_malloc_zero(sizeof(tor_cert_t));
+  cert->encoded = tor_memdup(encoded, len);
+  cert->encoded_len = len;
+
+  memcpy(cert->signed_key.pubkey, parsed->certified_key, 32);
+  int64_t valid_until_64 = ((int64_t)parsed->exp_field) * 3600;
+#if SIZEOF_TIME_T < 8
+  if (valid_until_64 > TIME_MAX)
+    valid_until_64 = TIME_MAX - 1;
+#endif
+  cert->valid_until = (time_t) valid_until_64;
+  cert->cert_type = parsed->cert_type;
+
+  for (unsigned i = 0; i < ed25519_cert_getlen_ext(parsed); ++i) {
+    ed25519_cert_extension_t *ext = ed25519_cert_get_ext(parsed, i);
+    if (ext->ext_type == CERTEXT_SIGNED_WITH_KEY) {
+      if (cert->signing_key_included)
+        goto err;
+
+      cert->signing_key_included = 1;
+      memcpy(cert->signing_key.pubkey, ext->un_signing_key, 32);
+    } else if (ext->ext_flags & CERTEXT_FLAG_AFFECTS_VALIDATION) {
+      /* Unrecognized extension with affects_validation set */
+      goto err;
+    }
+  }
+
+  goto done;
+ err:
+  tor_cert_free(cert);
+  cert = NULL;
+ done:
+  ed25519_cert_free(parsed);
+  return cert;
+}
+
+/** Fill in <b>checkable_out</b> with the information needed to check
+ * the signature on <b>cert</b> with <b>pubkey</b>.
+ *
+ * On success, if <b>expiration_out</b> is provided, and it is some time
+ * _after_ the expiration time of this certificate, set it to the
+ * expiration time of this certificate.
+ */
+int
+tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
+                           const tor_cert_t *cert,
+                           const ed25519_public_key_t *pubkey,
+                           time_t *expiration_out)
+{
+  if (! pubkey) {
+    if (cert->signing_key_included)
+      pubkey = &cert->signing_key;
+    else
+      return -1;
+  }
+
+  checkable_out->msg = cert->encoded;
+  checkable_out->pubkey = pubkey;
+  tor_assert(cert->encoded_len > ED25519_SIG_LEN);
+  const size_t signed_len = cert->encoded_len - ED25519_SIG_LEN;
+  checkable_out->len = signed_len;
+  memcpy(checkable_out->signature.sig,
+         cert->encoded + signed_len, ED25519_SIG_LEN);
+
+  if (expiration_out) {
+    *expiration_out = MIN(*expiration_out, cert->valid_until);
+  }
+
+  return 0;
+}
+
+/** Validates the signature on <b>cert</b> with <b>pubkey</b> relative to the
+ * current time <b>now</b>.  (If <b>now</b> is 0, do not check the expiration
+ * time.) Return 0 on success, -1 on failure.  Sets flags in <b>cert</b> as
+ * appropriate.
+ */
+int
+tor_cert_checksig(tor_cert_t *cert,
+                  const ed25519_public_key_t *pubkey, time_t now)
+{
+  ed25519_checkable_t checkable;
+  int okay;
+  time_t expires = TIME_MAX;
+
+  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey, &expires) < 0)
+    return -1;
+
+  if (now && now > expires) {
+    cert->cert_expired = 1;
+    return -1;
+  }
+
+  if (ed25519_checksig_batch(&okay, &checkable, 1) < 0) {
+    cert->sig_bad = 1;
+    return -1;
+  } else {
+    cert->sig_ok = 1;
+    /* Only copy the checkable public key when it is different from the signing
+     * key of the certificate to avoid undefined behavior. */
+    if (cert->signing_key.pubkey != checkable.pubkey->pubkey) {
+      memcpy(cert->signing_key.pubkey, checkable.pubkey->pubkey, 32);
+    }
+    cert->cert_valid = 1;
+    return 0;
+  }
+}
+
+/** Return a string describing the status of the signature on <b>cert</b>
+ *
+ * Will always be "unchecked" unless tor_cert_checksig has been called.
+ */
+const char *
+tor_cert_describe_signature_status(const tor_cert_t *cert)
+{
+  if (cert->cert_expired) {
+    return "expired";
+  } else if (cert->sig_bad) {
+    return "mis-signed";
+  } else if (cert->sig_ok) {
+    return "okay";
+  } else {
+    return "unchecked";
+  }
+}
+
+/** Return a new copy of <b>cert</b> */
+tor_cert_t *
+tor_cert_dup(const tor_cert_t *cert)
+{
+  tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
+  if (cert->encoded)
+    newcert->encoded = tor_memdup(cert->encoded, cert->encoded_len);
+  return newcert;
+}
+
+/** Return true iff cert1 and cert2 are the same cert. */
+int
+tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
+{
+  tor_assert(cert1);
+  tor_assert(cert2);
+  return cert1->encoded_len == cert2->encoded_len &&
+    tor_memeq(cert1->encoded, cert2->encoded, cert1->encoded_len);
+}
+
+/** Return true iff cert1 and cert2 are the same cert, or if they are both
+ * NULL. */
+int
+tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
+{
+  if (cert1 == NULL && cert2 == NULL)
+    return 1;
+  if (!cert1 || !cert2)
+    return 0;
+  return tor_cert_eq(cert1, cert2);
+}
+
+#define RSA_ED_CROSSCERT_PREFIX "Tor TLS RSA/Ed25519 cross-certificate"
+
+/** Create new cross-certification object to certify <b>ed_key</b> as the
+ * master ed25519 identity key for the RSA identity key <b>rsa_key</b>.
+ * Allocates and stores the encoded certificate in *<b>cert</b>, and returns
+ * the number of bytes stored. Returns negative on error.*/
+ssize_t
+tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
+                               const crypto_pk_t *rsa_key,
+                               time_t expires,
+                               uint8_t **cert)
+{
+  // It is later than 1985, since otherwise there would be no C89
+  // compilers. (Try to diagnose #22466.)
+  tor_assert_nonfatal(expires >= 15 * 365 * 86400);
+
+  uint8_t *res;
+
+  rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new();
+  memcpy(cc->ed_key, ed_key->pubkey, ED25519_PUBKEY_LEN);
+  cc->expiration = (uint32_t) CEIL_DIV(expires, 3600);
+  cc->sig_len = crypto_pk_keysize(rsa_key);
+  rsa_ed_crosscert_setlen_sig(cc, crypto_pk_keysize(rsa_key));
+
+  ssize_t alloc_sz = rsa_ed_crosscert_encoded_len(cc);
+  tor_assert(alloc_sz > 0);
+  res = tor_malloc_zero(alloc_sz);
+  ssize_t sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
+  tor_assert(sz > 0 && sz <= alloc_sz);
+
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+
+  const int signed_part_len = 32 + 4;
+  crypto_digest_add_bytes(d, (char*)res, signed_part_len);
+
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  int siglen = crypto_pk_private_sign(rsa_key,
+                                      (char*)rsa_ed_crosscert_getarray_sig(cc),
+                                      rsa_ed_crosscert_getlen_sig(cc),
+                                      (char*)digest, sizeof(digest));
+  tor_assert(siglen > 0 && siglen <= (int)crypto_pk_keysize(rsa_key));
+  tor_assert(siglen <= UINT8_MAX);
+  cc->sig_len = siglen;
+  rsa_ed_crosscert_setlen_sig(cc, siglen);
+
+  sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
+  rsa_ed_crosscert_free(cc);
+  *cert = res;
+  return sz;
+}
+
+/**
+ * Check whether the <b>crosscert_len</b> byte certificate in <b>crosscert</b>
+ * is in fact a correct cross-certification of <b>master_key</b> using
+ * the RSA key <b>rsa_id_key</b>.
+ *
+ * Also reject the certificate if it expired before
+ * <b>reject_if_expired_before</b>.
+ *
+ * Return 0 on success, negative on failure.
+ */
+MOCK_IMPL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before))
+{
+  rsa_ed_crosscert_t *cc = NULL;
+  int rv;
+
+#define ERR(code, s)                                            \
+  do {                                                          \
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,                      \
+           "Received a bad RSA->Ed25519 crosscert: %s",         \
+           (s));                                                \
+    rv = (code);                                                \
+    goto err;                                                   \
+  } while (0)
+
+  if (BUG(crypto_pk_keysize(rsa_id_key) > PK_BYTES))
+    return -1;
+
+  if (BUG(!crosscert))
+    return -1;
+
+  ssize_t parsed_len = rsa_ed_crosscert_parse(&cc, crosscert, crosscert_len);
+  if (parsed_len < 0 || crosscert_len != (size_t)parsed_len) {
+    ERR(-2, "Unparseable or overlong crosscert");
+  }
+
+  if (tor_memneq(rsa_ed_crosscert_getarray_ed_key(cc),
+                 master_key->pubkey,
+                 ED25519_PUBKEY_LEN)) {
+    ERR(-3, "Crosscert did not match Ed25519 key");
+  }
+
+  const uint32_t expiration_date = rsa_ed_crosscert_get_expiration(cc);
+  const uint64_t expiration_time = ((uint64_t)expiration_date) * 3600;
+
+  if (reject_if_expired_before < 0 ||
+      expiration_time < (uint64_t)reject_if_expired_before) {
+    ERR(-4, "Crosscert is expired");
+  }
+
+  const uint8_t *eos = rsa_ed_crosscert_get_end_of_signed(cc);
+  const uint8_t *sig = rsa_ed_crosscert_getarray_sig(cc);
+  const uint8_t siglen = rsa_ed_crosscert_get_sig_len(cc);
+  tor_assert(eos >= crosscert);
+  tor_assert((size_t)(eos - crosscert) <= crosscert_len);
+  tor_assert(siglen == rsa_ed_crosscert_getlen_sig(cc));
+
+  /* Compute the digest */
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+  crypto_digest_add_bytes(d, (char*)crosscert, eos-crosscert);
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  /* Now check the signature */
+  uint8_t signed_[PK_BYTES];
+  int signed_len = crypto_pk_public_checksig(rsa_id_key,
+                                          (char*)signed_, sizeof(signed_),
+                                          (char*)sig, siglen);
+  if (signed_len < DIGEST256_LEN) {
+    ERR(-5, "Bad signature, or length of signed data not as expected");
+  }
+
+  if (tor_memneq(digest, signed_, DIGEST256_LEN)) {
+    ERR(-6, "The signature was good, but it didn't match the data");
+  }
+
+  rv = 0;
+ err:
+  rsa_ed_crosscert_free(cc);
+  return rv;
+}
+
+/** Construct and return a new empty or_handshake_certs object */
+or_handshake_certs_t *
+or_handshake_certs_new(void)
+{
+  return tor_malloc_zero(sizeof(or_handshake_certs_t));
+}
+
+/** Release all storage held in <b>certs</b> */
+void
+or_handshake_certs_free_(or_handshake_certs_t *certs)
+{
+  if (!certs)
+    return;
+
+  tor_x509_cert_free(certs->auth_cert);
+  tor_x509_cert_free(certs->link_cert);
+  tor_x509_cert_free(certs->id_cert);
+
+  tor_cert_free(certs->ed_id_sign);
+  tor_cert_free(certs->ed_sign_link);
+  tor_cert_free(certs->ed_sign_auth);
+  tor_free(certs->ed_rsa_crosscert);
+
+  memwipe(certs, 0xBD, sizeof(*certs));
+  tor_free(certs);
+}
+
+#undef ERR
+#define ERR(s)                                                  \
+  do {                                                          \
+    log_fn(severity, LD_PROTOCOL,                               \
+           "Received a bad CERTS cell: %s",                     \
+           (s));                                                \
+    return 0;                                                   \
+  } while (0)
+
+int
+or_handshake_certs_rsa_ok(int severity,
+                          or_handshake_certs_t *certs,
+                          tor_tls_t *tls,
+                          time_t now)
+{
+  tor_x509_cert_t *link_cert = certs->link_cert;
+  tor_x509_cert_t *auth_cert = certs->auth_cert;
+  tor_x509_cert_t *id_cert = certs->id_cert;
+
+  if (certs->started_here) {
+    if (! (id_cert && link_cert))
+      ERR("The certs we wanted (ID, Link) were missing");
+    if (! tor_tls_cert_matches_key(tls, link_cert))
+      ERR("The link certificate didn't match the TLS public key");
+    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, now, 0))
+      ERR("The link certificate was not valid");
+    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  } else {
+    if (! (id_cert && auth_cert))
+      ERR("The certs we wanted (ID, Auth) were missing");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, now, 1))
+      ERR("The authentication certificate was not valid");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  }
+
+  return 1;
+}
+
+/** Check all the ed25519 certificates in <b>certs</b> against each other, and
+ * against the peer certificate in <b>tls</b> if appropriate.  On success,
+ * return 0; on failure, return a negative value and warn at level
+ * <b>severity</b> */
+int
+or_handshake_certs_ed25519_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now)
+{
+  ed25519_checkable_t check[10];
+  unsigned n_checkable = 0;
+  time_t expiration = TIME_MAX;
+
+#define ADDCERT(cert, pk)                                               \
+  do {                                                                  \
+    tor_assert(n_checkable < ARRAY_LENGTH(check));                      \
+    if (tor_cert_get_checkable_sig(&check[n_checkable++], cert, pk,     \
+                                   &expiration) < 0)                    \
+      ERR("Could not get checkable cert.");                             \
+  } while (0)
+
+  if (! certs->ed_id_sign || !certs->ed_id_sign->signing_key_included) {
+    ERR("No Ed25519 signing key");
+  }
+  ADDCERT(certs->ed_id_sign, NULL);
+
+  if (certs->started_here) {
+    if (! certs->ed_sign_link)
+      ERR("No Ed25519 link key");
+    {
+      /* check for a match with the TLS cert. */
+      tor_x509_cert_t *peer_cert = tor_tls_get_peer_cert(tls);
+      if (BUG(!peer_cert)) {
+        /* This is a bug, because if we got to this point, we are a connection
+         * that was initiated here, and we completed a TLS handshake. The
+         * other side *must* have given us a certificate! */
+        ERR("No x509 peer cert"); // LCOV_EXCL_LINE
+      }
+      const common_digests_t *peer_cert_digests =
+        tor_x509_cert_get_cert_digests(peer_cert);
+      int okay = tor_memeq(peer_cert_digests->d[DIGEST_SHA256],
+                           certs->ed_sign_link->signed_key.pubkey,
+                           DIGEST256_LEN);
+      tor_x509_cert_free(peer_cert);
+      if (!okay)
+        ERR("Link certificate does not match TLS certificate");
+    }
+
+    ADDCERT(certs->ed_sign_link, &certs->ed_id_sign->signed_key);
+
+  } else {
+    if (! certs->ed_sign_auth)
+      ERR("No Ed25519 link authentication key");
+    ADDCERT(certs->ed_sign_auth, &certs->ed_id_sign->signed_key);
+  }
+
+  if (expiration < now) {
+    ERR("At least one certificate expired.");
+  }
+
+  /* Okay, we've gotten ready to check all the Ed25519 certificates.
+   * Now, we are going to check the RSA certificate's cross-certification
+   * with the ED certificates.
+   *
+   * FFFF In the future, we might want to make this optional.
+   */
+
+  tor_x509_cert_t *rsa_id_cert = certs->id_cert;
+  if (!rsa_id_cert) {
+    ERR("Missing legacy RSA ID certificate");
+  }
+  if (! tor_tls_cert_is_valid(severity, rsa_id_cert, rsa_id_cert, now, 1)) {
+    ERR("The legacy RSA ID certificate was not valid");
+  }
+  if (! certs->ed_rsa_crosscert) {
+    ERR("Missing RSA->Ed25519 crosscert");
+  }
+  crypto_pk_t *rsa_id_key = tor_tls_cert_get_key(rsa_id_cert);
+  if (!rsa_id_key) {
+    ERR("RSA ID cert had no RSA key");
+  }
+
+  if (rsa_ed25519_crosscert_check(certs->ed_rsa_crosscert,
+                                  certs->ed_rsa_crosscert_len,
+                                  rsa_id_key,
+                                  &certs->ed_id_sign->signing_key,
+                                  now) < 0) {
+    crypto_pk_free(rsa_id_key);
+    ERR("Invalid RSA->Ed25519 crosscert");
+  }
+  crypto_pk_free(rsa_id_key);
+  rsa_id_key = NULL;
+
+  /* FFFF We could save a little time in the client case by queueing
+   * this batch to check it later, along with the signature from the
+   * AUTHENTICATE cell. That will change our data flow a bit, though,
+   * so I say "postpone". */
+
+  if (ed25519_checksig_batch(NULL, check, n_checkable) < 0) {
+    ERR("At least one Ed25519 certificate was badly signed");
+  }
+
+  return 1;
+}
+
+/**
+ * Check the Ed certificates and/or the RSA certificates, as appropriate.  If
+ * we obtained an Ed25519 identity, set *ed_id_out. If we obtained an RSA
+ * identity, set *rs_id_out. Otherwise, set them both to NULL.
+ */
+void
+or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out)
+{
+  tor_assert(ed_id_out);
+  tor_assert(rsa_id_out);
+
+  *ed_id_out = NULL;
+  *rsa_id_out = NULL;
+
+  if (certs->ed_id_sign) {
+    if (or_handshake_certs_ed25519_ok(severity, certs, tls, now)) {
+      tor_assert(certs->ed_id_sign);
+      tor_assert(certs->id_cert);
+
+      *ed_id_out = &certs->ed_id_sign->signing_key;
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+
+      /* If we reached this point, we did not look at any of the
+       * subsidiary RSA certificates, so we'd better just remove them.
+       */
+      tor_x509_cert_free(certs->link_cert);
+      tor_x509_cert_free(certs->auth_cert);
+      certs->link_cert = certs->auth_cert = NULL;
+    }
+    /* We do _not_ fall through here.  If you provided us Ed25519
+     * certificates, we expect to verify them! */
+  } else {
+    /* No ed25519 keys given in the CERTS cell */
+    if (or_handshake_certs_rsa_ok(severity, certs, tls, now)) {
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+    }
+  }
+}
+
+/* === ENCODING === */
+
+/* Encode the ed25519 certificate <b>cert</b> and put the newly allocated
+ * string in <b>cert_str_out</b>. Return 0 on success else a negative value. */
+int
+tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out)
+{
+  int ret = -1;
+  char *ed_cert_b64 = NULL;
+  size_t ed_cert_b64_len;
+
+  tor_assert(cert);
+  tor_assert(cert_str_out);
+
+  /* Get the encoded size and add the NUL byte. */
+  ed_cert_b64_len = base64_encode_size(cert->encoded_len,
+                                       BASE64_ENCODE_MULTILINE) + 1;
+  ed_cert_b64 = tor_malloc_zero(ed_cert_b64_len);
+
+  /* Base64 encode the encoded certificate. */
+  if (base64_encode(ed_cert_b64, ed_cert_b64_len,
+                    (const char *) cert->encoded, cert->encoded_len,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    /* LCOV_EXCL_START */
+    log_err(LD_BUG, "Couldn't base64-encode ed22519 cert!");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Put everything together in a NUL terminated string. */
+  tor_asprintf(cert_str_out,
+               "-----BEGIN ED25519 CERT-----\n"
+               "%s"
+               "-----END ED25519 CERT-----",
+               ed_cert_b64);
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(ed_cert_b64);
+  return ret;
+}
diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h
new file mode 100644
index 0000000000..5fa97679df
--- /dev/null
+++ b/src/feature/nodelist/torcert.h
@@ -0,0 +1,110 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TORCERT_H_INCLUDED
+#define TORCERT_H_INCLUDED
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#define SIGNED_KEY_TYPE_ED25519     0x01
+
+#define CERT_TYPE_ID_SIGNING        0x04
+#define CERT_TYPE_SIGNING_LINK      0x05
+#define CERT_TYPE_SIGNING_AUTH      0x06
+#define CERT_TYPE_SIGNING_HS_DESC   0x08
+#define CERT_TYPE_AUTH_HS_IP_KEY    0x09
+#define CERT_TYPE_ONION_ID          0x0A
+#define CERT_TYPE_CROSS_HS_IP_KEYS  0x0B
+
+#define CERT_FLAG_INCLUDE_SIGNING_KEY 0x1
+
+/** An ed25519-signed certificate as used throughout the Tor protocol.
+ **/
+typedef struct tor_cert_st {
+  /** The key authenticated by this certificate */
+  ed25519_public_key_t signed_key;
+  /** The key that signed this certificate. This value may be unset if the
+   * certificate has never been checked, and didn't include its own key. */
+  ed25519_public_key_t signing_key;
+  /** A time after which this certificate will no longer be valid. */
+  time_t valid_until;
+
+  /** The encoded representation of this certificate */
+  uint8_t *encoded;
+  /** The length of <b>encoded</b> */
+  size_t encoded_len;
+
+  /** One of CERT_TYPE_... */
+  uint8_t cert_type;
+  /** True iff we received a signing key embedded in this certificate */
+  unsigned signing_key_included : 1;
+  /** True iff we checked the signature and found it bad */
+  unsigned sig_bad : 1;
+  /** True iff we checked the signature and found it correct */
+  unsigned sig_ok : 1;
+  /** True iff we checked the signature and first found that the cert
+   * had expired */
+  unsigned cert_expired : 1;
+  /** True iff we checked the signature and found the whole cert valid */
+  unsigned cert_valid : 1;
+} tor_cert_t;
+
+struct tor_tls_t;
+
+tor_cert_t *tor_cert_create(const ed25519_keypair_t *signing_key,
+                            uint8_t cert_type,
+                            const ed25519_public_key_t *signed_key,
+                            time_t now, time_t lifetime,
+                            uint32_t flags);
+
+tor_cert_t *tor_cert_parse(const uint8_t *cert, size_t certlen);
+
+void tor_cert_free_(tor_cert_t *cert);
+#define tor_cert_free(cert) FREE_AND_NULL(tor_cert_t, tor_cert_free_, (cert))
+
+int tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
+                               const tor_cert_t *out,
+                               const ed25519_public_key_t *pubkey,
+                               time_t *expiration_out);
+
+int tor_cert_checksig(tor_cert_t *cert,
+                      const ed25519_public_key_t *pubkey, time_t now);
+const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
+
+tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
+int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
+int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
+
+ssize_t tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
+                                       const crypto_pk_t *rsa_key,
+                                       time_t expires,
+                                       uint8_t **cert);
+MOCK_DECL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before));
+
+or_handshake_certs_t *or_handshake_certs_new(void);
+void or_handshake_certs_free_(or_handshake_certs_t *certs);
+#define or_handshake_certs_free(certs) \
+  FREE_AND_NULL(or_handshake_certs_t, or_handshake_certs_free_, (certs))
+int or_handshake_certs_rsa_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              struct tor_tls_t *tls,
+                              time_t now);
+int or_handshake_certs_ed25519_ok(int severity,
+                                  or_handshake_certs_t *certs,
+                                  struct tor_tls_t *tls,
+                                  time_t now);
+void or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              struct tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out);
+
+int tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out);
+
+#endif /* !defined(TORCERT_H_INCLUDED) */
diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h
new file mode 100644
index 0000000000..1b85737df8
--- /dev/null
+++ b/src/feature/nodelist/vote_routerstatus_st.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VOTE_ROUTERSTATUS_ST_H
+#define VOTE_ROUTERSTATUS_ST_H
+
+#include "or/routerstatus_st.h"
+#include "lib/defs/x25519_sizes.h"
+
+/** The claim about a single router, made in a vote. */
+struct vote_routerstatus_t {
+  routerstatus_t status; /**< Underlying 'status' object for this router.
+                          * Flags are redundant. */
+  /** How many known-flags are allowed in a vote? This is the width of
+   * the flags field of vote_routerstatus_t */
+#define MAX_KNOWN_FLAGS_IN_VOTE 64
+  uint64_t flags; /**< Bit-field for all recognized flags; index into
+                   * networkstatus_t.known_flags. */
+  char *version; /**< The version that the authority says this router is
+                  * running. */
+  char *protocols; /**< The protocols that this authority says this router
+                    * provides. */
+  unsigned int has_measured_bw:1; /**< The vote had a measured bw */
+  /** True iff the vote included an entry for ed25519 ID, or included
+   * "id ed25519 none" to indicate that there was no ed25519 ID. */
+  unsigned int has_ed25519_listing:1;
+  /** True if the Ed25519 listing here is the consensus-opinion for the
+   * Ed25519 listing; false if there was no consensus on Ed25519 key status,
+   * or if this VRS doesn't reflect it. */
+  unsigned int ed25519_reflects_consensus:1;
+  uint32_t measured_bw_kb; /**< Measured bandwidth (capacity) of the router */
+  /** The hash or hashes that the authority claims this microdesc has. */
+  vote_microdesc_hash_t *microdesc;
+  /** Ed25519 identity for this router, or zero if it has none. */
+  uint8_t ed25519_id[ED25519_PUBKEY_LEN];
+};
+
+#endif